File Coverage

File:blib/lib/App/FileSync.pm
Coverage:92.8%

linestmtbrancondsubtimecode
1package App::FileSync;
2
3# The pieces shared by the client, the server and the admin tools:
4# byte-transparent JSON, path and id validation, conflict-copy naming, date
5# parsing and (re-exported from App::FileSync::IO) atomic file writes.
6
7
318
318
318
287815
281
4504
use App::FileSync::Platform;
8
9
318
318
318
550
252
2540
use strict;
10
318
318
318
483
215
5689
use warnings;
11
318
318
318
1082
18269
1447
use autodie qw(:all);
12
318
318
318
700762
499
20180
use feature 'current_sub';
13
14
318
318
318
935
384
3815
use Exporter qw(import);
15
318
318
318
109828
1965498
4022
use JSON::PP ();
16
318
318
318
1510
5246
8342
use Readonly;
17
318
318
318
1323
78056
1182
use Sub::Private { mode => 'enforce' };
18
318
318
318
100030
231590
5096
use Time::Local ();
19
20
318
318
318
45385
384
23326
use App::FileSync::Constants qw($MAX_ID_LENGTH $SECONDS_PER_DAY);
21
318
318
318
52954
412
10632
use App::FileSync::I18N qw(message throw);
22
318
318
318
52613
511
10438
use App::FileSync::IO qw(write_atomic read_text set_mtime);
23
24
318
318
318
782
259
607
use version; our $VERSION = version->declare('0.001.1');
25
26our @EXPORT_OK = qw(
27        json_encode json_decode
28        validate_relpath validate_repo_name validate_client_id
29        conflict_name name_with_suffix split_name
30        format_utc parse_since parse_until
31        write_atomic read_text set_mtime
32);
33
34# Limits on paths as they travel on the wire.  Package variables rather than
35# Readonly because they document the protocol and tests read them.
36our $MAX_PATH_BYTES      = 4096;
37our $MAX_COMPONENT_BYTES = 255;
38
39# A conflict-copy name is kept this far inside the common 255-byte limit, and
40# never trims the original stem below the minimum.
41Readonly my $CONFLICT_NAME_BYTES => 200;
42Readonly my $MIN_STEM_BYTES      => 16;
43
44# The alphabet of a repo or client id: it becomes a directory name.
45Readonly my $ID_PATTERN => qr/\A[A-Za-z0-9][A-Za-z0-9._-]*\z/;
46
47# What an unusable client id is replaced with in a conflict-copy name.
48Readonly my $UNKNOWN_CLIENT => 'unknown';
49
50# Deliberately utf8(0): paths are byte strings on the wire, and both ends of the
51# protocol are this implementation.
52my $JSON = JSON::PP->new->canonical->allow_nonref;
53
54# JSON::PP returns UTF8-flagged strings even though paths are byte strings on
55# disk.  Left alone, the flag makes -e/stat encode the name a second time and
56# every non-ASCII file "disappears".  This downgrades back to the original
57# bytes; only a genuine wide-character string is encoded as UTF-8.
58#
59# A lexical coderef rather than a named sub: it runs once per JSON value (six
60# times per manifest entry), and the Sub::Private enforce wrapper would cost
61# more than the work.  Being lexical, nothing outside this file can reach it.
62my $to_bytes = sub {
63        # Work through @_ aliases: copying the scalar would rewrite the copy only.
64        if (ref($_[0]) eq 'HASH') {
65                __SUB__->($_) for values %{ $_[0] };
66        } elsif (ref($_[0]) eq 'ARRAY') {
67                __SUB__->($_) for @{ $_[0] };
68        } elsif (defined($_[0]) && !ref($_[0]) && utf8::is_utf8($_[0])) {
69                utf8::encode($_[0]) unless utf8::downgrade($_[0], 1);
70        }
71        return 1;
72};
73
74=encoding utf8
75
76 - 150
=head1 NAME

App::FileSync - cron-driven file synchronisation without a daemon

=head1 VERSION

0.001.1

=head1 SYNOPSIS

        filesync init --repo documents --root ~/Documents \
                --transport local --server-root /srv/filesync --remote-repo documents
        filesync sync --repo documents --dry-run
        filesync sync --repo documents

=head1 DESCRIPTION

App::FileSync keeps one directory per client in step with a master copy on a server, and
keeps a separate machine holding a replica of that master.  Every run is a short-lived CLI
process suitable for cron or the Windows Task Scheduler: there is no daemon, no database
and no web interface.

Only files whose content changed are transferred.  Data frames are deflated when that
makes them smaller, and files of at least 64 KiB are sent as rsync-style block deltas
when the other side already holds an older copy, so an edit moves the changed blocks
rather than the whole file.  C<filesync sync --no-delta> forces whole files.  Each run
reports the payload bytes it sent and received and how many files moved as a delta, and
C<filesync log> repeats those figures for past runs.

Every message the tools print is looked up in a catalogue
(L<App::FileSync::I18N>), so a translation is one new catalogue.

This module holds the pieces shared by the client, the server and the admin tools:
byte-transparent JSON, path validation, conflict-copy naming and atomic file writes.

=head1 FUNCTIONS

=head2 json_encode

Purpose: encode a structure as one line of canonical JSON.

Args: C<$data>, any JSON-encodable value.

Returns: the JSON text, keys sorted, with no trailing newline.

Side Effects: none.

Usage:

        print {$fh} json_encode($event), "\n";

=head3 EXAMPLE

        print json_encode({ b => 1, a => [ 'x' ] }), "\n";      # {"a":["x"],"b":1}

=head3 API SPECIFICATION

=head4 INPUT

        [ { name => 'data', type => 'scalar', optional => 1 } ]

=head4 OUTPUT

        { type => 'string', min => 1 }

=head3 MESSAGES

JSON::PP's own error for a value it cannot encode (a code reference, say).

=head3 FORMAL SPECIFICATION

        json_encode : VALUE → TEXT
        json_decode(json_encode(v)) = v

=cut
151
152sub json_encode {
153
4481
13805
        my ($data) = @_;
154
155
4481
7857
        return $JSON->encode($data);
156}
157
158 - 197
=head2 json_decode

Purpose: decode one JSON document, keeping strings as byte strings.

Args: C<$text>.

Returns: the decoded structure.  Strings that fit in bytes are downgraded, so a
path read back from JSON names the same file as the bytes it was written from.

Side Effects: none.

Usage:

        my $config = json_decode(read_text($path));

=head3 EXAMPLE

        my $entry = json_decode('{"p":"caf\u00e9.txt"}');
        print length($entry->{p}), "\n";   # 8 (bytes), not 9

=head3 API SPECIFICATION

=head4 INPUT

        [ { name => 'text', type => 'string' } ]

=head4 OUTPUT

        { type => 'scalar', optional => 1 }

=head3 MESSAGES

JSON::PP's own error for malformed JSON.

=head3 FORMAL SPECIFICATION

        json_decode : TEXT ↛ VALUE
        âˆ€ s : STRING in result • ¬ utf8_flag(s) ∨ ∃ c ∈ s • ord c > 255

=cut
198
199sub json_decode {
200
5021
104740374
        my ($text) = @_;
201
202
5021
9550
        my $data = $JSON->decode($text);
203
5018
4376179
        $to_bytes->($data);
204
205
5018
6538
        return $data;
206}
207
208 - 277
=head2 validate_relpath

Purpose: decide whether a path is safe to join to a sync root.

Args: C<$path>, a relative path as it travels on the wire.

Returns: a two-element list C<(1, undef)> for a usable path, or
C<(0, $reason)> where C<$reason> is a translated explanation.  (A list rather
than a reference because every caller destructures it at once, and it is
called once per path in the tree.)

Side Effects: none.

Usage:

        my ($ok, $reason) = validate_relpath($path);

=head3 EXAMPLE

        my ($ok, $why) = validate_relpath('notes/../secret');
        print "$why\n" unless $ok;    # parent path component

=head3 API SPECIFICATION

=head4 INPUT

        [ { name => 'path', type => 'scalar', optional => 1 } ]

=head4 OUTPUT

        { type => 'arrayref', min => 2, max => 2 }

=head3 MESSAGES

        Key                       Meaning
        path.missing              no path was given
        path.not_text             the path is a reference
        path.empty                the path is empty
        path.too_long             longer than 4096 bytes
        path.absolute             starts with /
        path.drive                starts with a drive letter
        path.backslash            contains a backslash
        path.nul                  contains a NUL byte
        path.control              contains a control character
        path.empty_component      contains //, or starts or ends with /
        path.dot_component        has a . component
        path.parent_component     has a .. component
        path.component_too_long   a component is longer than 255 bytes

Resolution: rename the file on the side that holds it.

=head3 FORMAL SPECIFICATION

        SafePath == { p : seq BYTE |
                0 < #p ≤ 4096 ∧ head p ≠ '/' ∧ ¬ drive(p) ∧
                '\' ∉ ran p ∧ ran p ∩ CONTROL = ∅ ∧
                âˆ€ c : split('/', p) • c ∉ {⟨⟩, ".", ".."} ∧ #c ≤ 255 }
        validate_relpath(p) = (1, ⊥)        if p ∈ SafePath
                            = (0, reason)   otherwise

=head3 PSEUDOCODE

        reject undef, references and empty strings
        reject paths over 4096 bytes
        reject absolute, drive-relative, backslash, NUL and control characters
        for each "/"-separated component:
                reject empty, ".", ".." and components over 255 bytes
        accept

=cut
278
279sub validate_relpath {
280
1678
82154
        my ($path) = @_;
281
282        # One key per rejection, checked in order: the first problem found is
283        # the one reported.
284
1678
6673
        my $problem =
285                  !defined $path                       ? 'path.missing'
286                : ref $path                            ? 'path.not_text'
287                : !length $path                        ? 'path.empty'
288                : length($path) > $MAX_PATH_BYTES      ? 'path.too_long'
289                : $path =~ m{\A/}                      ? 'path.absolute'
290                : $path =~ /\A[A-Za-z]:/               ? 'path.drive'
291                : $path =~ /\\/                        ? 'path.backslash'
292                : $path =~ /\0/                        ? 'path.nul'
293                : $path =~ /[\x01-\x1f\x7f]/           ? 'path.control'
294                : undef;
295
296        # Components are only worth checking once the whole path is plausible.
297
1678
1665
        unless (defined $problem) {
298
1665
2358
                for my $part (split m{/}, $path, -1) {
299
1883
3001
                        $problem =
300                                  $part eq ''                             ? 'path.empty_component'
301                                : $part eq '.'                            ? 'path.dot_component'
302                                : $part eq '..'                           ? 'path.parent_component'
303                                : length($part) > $MAX_COMPONENT_BYTES    ? 'path.component_too_long'
304                                : undef;
305
1883
1747
                        last if defined $problem;
306                }
307        }
308
309
1678
2509
        return defined $problem ? (0, message($problem)) : (1, undef);
310}
311
312 - 360
=head2 validate_repo_name

Documented with L</validate_client_id> below.

=head2 validate_client_id

Purpose: decide whether a repo name or client id is usable as a directory name.

Args: C<$value>.

Returns: C<(1, undef)>, or C<(0, $reason)> with a translated reason.  The
alphabet is C<[A-Za-z0-9][A-Za-z0-9._-]*>, at most 64 characters.

Side Effects: none.

Usage:

        my ($ok, $reason) = validate_client_id($id);

=head3 EXAMPLE

        my ($ok, $why) = validate_repo_name('../etc');
        print "$why\n";       # bad repo name '../etc'

=head3 API SPECIFICATION

=head4 INPUT

        [ { name => 'value', type => 'scalar', optional => 1 } ]

=head4 OUTPUT

        { type => 'arrayref', min => 2, max => 2 }

=head3 MESSAGES

        Key           Meaning                         Resolution
        id.missing    no value was given              supply one
        id.not_text   the value is a reference        pass a string
        id.empty      the value is empty              supply one
        id.too_long   longer than 64 characters       shorten it
        id.bad        outside the allowed alphabet    use letters, digits, . _ -

=head3 FORMAL SPECIFICATION

        Id == { s : seq CHAR | 0 < #s ≤ 64 ∧ head s ∈ ALNUM ∧ ran s ⊆ ALNUM ∪ {'.','_','-'} }
        validate(s) = (1, ⊥) if s ∈ Id, (0, reason) otherwise

=cut
361
362sub validate_repo_name {
363
1212
3908
        my ($value) = @_;
364
365
1212
2136
        return _validate_id('id.repo_name', $value);
366}
367
368sub validate_client_id {
369
578
747
        my ($value) = @_;
370
371
578
955
        return _validate_id('id.client_id', $value);
372}
373
374# _validate_id
375# Purpose:     shared check for repo names and client ids.
376# Entry:       $what_key is the catalogue key naming the kind of id;
377#              $value is the candidate.
378# Exit:        returns (1, undef) or (0, translated reason).
379# Side effects: none.
380sub _validate_id :Private {
381        my ($what_key, $value) = @_;
382
383        my $problem =
384                  !defined $value                    ? 'id.missing'
385                : ref $value                         ? 'id.not_text'
386                : !length $value                     ? 'id.empty'
387                : length($value) > $MAX_ID_LENGTH    ? 'id.too_long'
388                : $value !~ $ID_PATTERN              ? 'id.bad'
389                : undef;
390
391        return defined $problem
392                ? (0, message($problem, { what => message($what_key), value => $value }))
393                : (1, undef);
394
318
318
318
148048
282
3632
}
395
396 - 439
=head2 parse_since

Purpose: the start of a time window.

Args: C<$value> (C<YYYY-MM-DD> or an epoch), optional C<$option> naming the
command-line option for the error message (default C<--since>).

Returns: the epoch, or C<undef> when C<$value> is undef.  A date means
midnight UTC that day.

Side Effects: none.

Usage:

        my $since = parse_since($opts{since});

=head3 EXAMPLE

        print parse_since('2026-10-22'), "\n";        # 1792627200

=head3 API SPECIFICATION

=head4 INPUT

        [
                { name => 'value',  type => 'string', optional => 1 },
                { name => 'option', type => 'string', optional => 1 },
        ]

=head4 OUTPUT

        { type => 'integer', optional => 1, min => 0 }

=head3 MESSAGES

        Key        Meaning                              Resolution
        when.bad   neither a date nor an epoch          use YYYY-MM-DD or digits

=head3 FORMAL SPECIFICATION

        parse_since : (DATE ∪ â„• ∪ {⊥}) ↛ â„• ∪ {⊥}
        parse_since(n) = n ; parse_since(d) = midnight_utc(d) ; parse_since(⊥) = ⊥

=cut
440
441sub parse_since {
442
34
1217
        my ($value, $option) = @_;
443
444
34
34
36
406
        my ($epoch) = @{ _parse_when($value, $option || '--since') };
445
30
54
        return $epoch;
446}
447
448 - 489
=head2 parse_until

Purpose: the end of a time window.

Args: as for L</parse_since> (default option name C<--until>).

Returns: the epoch, or C<undef>.  A date means the last second of that day, so
C<--until 2026-10-22> includes that day's runs.

Side Effects: none.

Usage:

        my $until = parse_until($opts{until});

=head3 EXAMPLE

        print parse_until('2026-10-22'), "\n";        # 1792713599

=head3 API SPECIFICATION

=head4 INPUT

        [
                { name => 'value',  type => 'string', optional => 1 },
                { name => 'option', type => 'string', optional => 1 },
        ]

=head4 OUTPUT

        { type => 'integer', optional => 1, min => 0 }

=head3 MESSAGES

        Key        Meaning                              Resolution
        when.bad   neither a date nor an epoch          use YYYY-MM-DD or digits

=head3 FORMAL SPECIFICATION

        parse_until(n) = n ; parse_until(d) = midnight_utc(d) + 86399 ; parse_until(⊥) = ⊥

=cut
490
491sub parse_until {
492
33
1852136
        my ($value, $option) = @_;
493
494
33
33
38
177
        my ($epoch, $is_date) = @{ _parse_when($value, $option || '--until') };
495
31
118
        return (defined $epoch && $is_date) ? $epoch + $SECONDS_PER_DAY - 1 : $epoch;
496}
497
498# _parse_when
499# Purpose:     parse "YYYY-MM-DD" or an epoch.
500# Entry:       $value may be undef; $option names the option for errors.
501# Exit:        returns [ $epoch_or_undef, $is_date ]; a date cutoff means a
502#              different instant at each end of a range, hence the flag.
503# Side effects: raises when.bad for anything else.
504sub _parse_when :Private {
505        my ($value, $option) = @_;
506
507        my $parsed =
508                  !defined $value                          ? [ undef, 0 ]
509                : $value =~ /\A\d+\z/                      ? [ $value + 0, 0 ]
510                : $value =~ /\A(\d{4})-(\d{2})-(\d{2})\z/  ? [ Time::Local::timegm(0, 0, 0, $3, $2 - 1, $1), 1 ]
511                : undef;
512
513        throw('when.bad', { option => $option, value => $value }) unless $parsed;
514        return $parsed;
515
318
318
318
127573
316
2846
}
516
517 - 554
=head2 format_utc

Purpose: a compact UTC timestamp.

Args: optional C<$epoch> (default now).

Returns: C<YYYYMMDDTHHMMSSZ>.

Side Effects: none.

Usage:

        my $stamp = format_utc($run->{ts});

=head3 EXAMPLE

        print format_utc(0), "\n";    # 19700101T000000Z

=head3 API SPECIFICATION

=head4 INPUT

        [ { name => 'epoch', type => 'number', optional => 1 } ]

=head4 OUTPUT

        { type => 'string', matches => qr/\A\d{8}T\d{6}Z\z/ }

=head3 MESSAGES

None.

=head3 FORMAL SPECIFICATION

        format_utc : â„• → STAMP
        format_utc(e) = yyyy mm dd 'T' HH MM SS 'Z' of gmtime(e)

=cut
555
556sub format_utc {
557
156
413
        my ($epoch) = @_;
558
559
156
335
        my @t = gmtime(defined $epoch ? $epoch : time);
560
156
749
        return sprintf('%04d%02d%02dT%02d%02d%02dZ',
561                $t[5] + 1900, $t[4] + 1, $t[3], $t[2], $t[1], $t[0]);
562}
563
564 - 603
=head2 split_name

Purpose: split a relative path into directory, stem and extension.

Args: C<$path>.

Returns: an arrayref C<[ $dir, $stem, $ext ]>; C<$dir> has no trailing slash
and is empty at the top level, C<$ext> includes its dot and is empty when there
is none.  A leading dot (C<.profile>) is part of the stem.

Side Effects: none.

Usage:

        my ($dir, $stem, $ext) = @{ split_name('a/b/report.txt') };

=head3 EXAMPLE

        my $parts = split_name('notes/plan.v2.txt');    # ['notes', 'plan.v2', '.txt']

=head3 API SPECIFICATION

=head4 INPUT

        [ { name => 'path', type => 'string', min => 1 } ]

=head4 OUTPUT

        { type => 'arrayref', min => 3, max => 3 }

=head3 MESSAGES

None.

=head3 FORMAL SPECIFICATION

        split_name(p) = (d, s, e) where p = d ⁀ "/" ⁀ s ⁀ e ∨ (d = ⟨⟩ ∧ p = s ⁀ e)
                        e = ⟨⟩ ∨ (head e = '.' ∧ '.' ∉ ran(tail e)) ; s ≠ ⟨⟩

=cut
604
605sub split_name {
606
26
24
        my ($path) = @_;
607
608
26
58
        my ($dir, $base) = $path =~ m{\A(.*)/([^/]+)\z} ? ($1, $2) : ('', $path);
609
26
95
        my ($stem, $ext) = $base =~ /\A(.+)(\.[^.]+)\z/ ? ($1, $2) : ($base, '');
610
611
26
64
        return [ $dir, $stem, $ext ];
612}
613
614 - 654
=head2 name_with_suffix

Purpose: make a name unique by numbering it before its extension.

Args: C<$path>, C<$suffix> (a number).

Returns: C<report-2.txt> for C<report.txt> and 2, keeping the directory.

Side Effects: none.

Usage:

        my $candidate = name_with_suffix($loser_name, $n);

=head3 EXAMPLE

        print name_with_suffix('a/b.txt', 3), "\n";   # a/b-3.txt

=head3 API SPECIFICATION

=head4 INPUT

        [
                { name => 'path',   type => 'string', min => 1 },
                { name => 'suffix', type => 'integer', min => 1 },
        ]

=head4 OUTPUT

        { type => 'string', min => 1 }

=head3 MESSAGES

None.

=head3 FORMAL SPECIFICATION

        let (d, s, e) = split_name(p)
        name_with_suffix(p, n) = join(d, s ⁀ "-" ⁀ n ⁀ e)

=cut
655
656sub name_with_suffix {
657
0
0
        my ($path, $suffix) = @_;
658
659
0
0
0
0
        my ($dir, $stem, $ext) = @{ split_name($path) };
660
0
0
        return _join_dir($dir, "$stem-$suffix$ext");
661}
662
663 - 707
=head2 conflict_name

Purpose: the name a losing version is kept under.

Args: C<$path>, C<$client_id> (may be undef), C<$epoch> (default now).

Returns: the original name with C<.conflict-CLIENT-TIMESTAMP> inserted before
the extension, kept within 200 bytes so it fits common filesystems.

Side Effects: none.

Usage:

        my $name = conflict_name('notes.txt', 'laptop-1', time);

=head3 EXAMPLE

        print conflict_name('report.txt', 'laptop-1', 1_792_664_100), "\n";
        # report.conflict-laptop-1-20261022T101500Z.txt

=head3 API SPECIFICATION

=head4 INPUT

        [
                { name => 'path',      type => 'string', min => 1 },
                { name => 'client_id', type => 'string', optional => 1 },
                { name => 'epoch',     type => 'number', optional => 1 },
        ]

=head4 OUTPUT

        { type => 'string', min => 1 }

=head3 MESSAGES

None.

=head3 FORMAL SPECIFICATION

        let (d, s, e) = split_name(p) ; id = sanitise(c) ; t = format_utc(when)
        let room = max(16, 200 − #(".conflict-" ⁀ id ⁀ "-" ⁀ t ⁀ e))
        conflict_name(p, c, when) = join(d, s[1..room] ⁀ ".conflict-" ⁀ id ⁀ "-" ⁀ t ⁀ e)

=cut
708
709sub conflict_name {
710
26
95
        my ($path, $client_id, $epoch) = @_;
711
712
26
26
24
37
        my ($dir, $stem, $ext) = @{ split_name($path) };
713
714        # The id lands in a file name, so anything outside the id alphabet is
715        # flattened rather than trusted.
716
26
41
        my $id = defined $client_id ? $client_id : $UNKNOWN_CLIENT;
717
26
33
        $id =~ s/[^A-Za-z0-9._-]+/_/g;
718
26
31
        $id = $UNKNOWN_CLIENT unless length $id;
719
720
26
54
        my $tail = ".conflict-$id-" . format_utc($epoch) . $ext;
721
722        # Keep the whole name inside the 255-byte limit of common filesystems.
723
26
47
        my $room = $CONFLICT_NAME_BYTES - length($tail);
724
26
74
        $room = $MIN_STEM_BYTES if $room < $MIN_STEM_BYTES;
725
26
78
        $stem = substr($stem, 0, $room) if length($stem) > $room;
726
727
26
58
        return _join_dir($dir, $stem . $tail);
728}
729
730# _join_dir
731# Purpose:     put a relative directory and a name back together.
732# Entry:       $dir may be empty (top level).
733# Exit:        returns "dir/name" or "name".
734# Side effects: none.
735sub _join_dir :Private {
736        my ($dir, $name) = @_;
737
738        return length($dir) ? "$dir/$name" : $name;
739
318
318
318
161270
268
2878
}
740
741 - 753
=head2 write_atomic

Documented with L</set_mtime> below.

=head2 read_text

Documented with L</set_mtime> below.

=head2 set_mtime

Re-exported from L<App::FileSync::IO>, where they are documented.

=cut
754
7551;
756