File Coverage

File:blib/lib/App/FileSync/IO.pm
Coverage:86.7%

linestmtbrancondsubtimecode
1package App::FileSync::IO;
2
3# Filesystem primitives shared by the client, the server, the trash and the
4# snapshots.  Each one used to be written out two to four times; keeping them
5# here means a fix (such as O_EXCL temporaries) lands everywhere at once.
6
7
327
327
327
20754
290
4665
use App::FileSync::Platform;
8
9
327
327
327
522
222
2601
use strict;
10
327
327
327
455
240
6043
use warnings;
11
327
327
327
1092
23434
759
use autodie qw(:all);
12
13
327
327
327
770680
413177
5533
use Digest::SHA ();
14
327
327
327
774
235
4650
use Exporter qw(import);
15
327
327
327
608
231
8891
use File::Basename qw(dirname);
16
327
327
327
67407
665071
4603
use File::Copy ();
17
327
327
327
754
236
9762
use File::Path qw(make_path);
18
327
327
327
601
250
2311
use File::Spec;
19
327
327
327
117692
2379025
4377
use File::Temp ();
20
327
327
327
1458
5338
7772
use Readonly;
21
327
327
327
1359
85717
1424
use Sub::Private { mode => 'enforce' };
22
327
327
327
38493
258
3095
use Time::HiRes ();
23
24
327
327
327
1690
277
16418
use App::FileSync::Constants qw($MODE_MASK $TEMP_PREFIX);
25
327
327
327
1215
258
6670
use App::FileSync::I18N qw(throw os_error strip_location);
26
27
327
327
327
551
246
911
use version; our $VERSION = version->declare('0.001.1');
28
29our @EXPORT_OK = qw(
30        read_text open_file temp_file write_atomic install_file clear_target
31        set_mtime sha256_file list_dir copy_preserving append_line ensure_dir
32);
33
34# File::Temp wants at least four X's; ten keep collisions out of reach even
35# with thousands of uploads in one directory.
36Readonly my $TEMP_TEMPLATE => $TEMP_PREFIX . ('X' x 10);
37
38# Catalogue key for each open mode, so the error names what was attempted.
39Readonly my %OPEN_KEY => (
40        '<'  => 'io.read',
41        '>'  => 'io.write',
42        '>>' => 'io.append',
43);
44
45# The permission bits a new file would get from open() under the current
46# umask.  File::Temp creates 0600 files, which would otherwise leak into the
47# master and every replica.
48Readonly my $CREATE_MODE => oct('666');
49
50# Windows has no POSIX permission bits worth setting.
51Readonly my $HAS_MODES => $^O ne 'MSWin32';
52
53=encoding utf8
54
55 - 120
=head1 NAME

App::FileSync::IO - filesystem primitives for App::FileSync

=head1 SYNOPSIS

        use App::FileSync::IO qw(write_atomic read_text sha256_file);

        write_atomic('/srv/filesync/.filesync/repos/docs/master.json', $json);
        my $text = read_text($path);

=head1 DESCRIPTION

Every function raises a catalogued error (see L<App::FileSync::I18N>) that
names the path and carries the operating system's reason.

=head1 FUNCTIONS

=head2 open_file

Purpose: open a file in binary mode.

Args: C<$path>, C<$mode> (C<< < >>, C<< > >> or C<<< >> >>>).

Returns: the filehandle.

Side Effects: creates or truncates the file for C<< > >>, creates it for
C<<< >> >>>.

Usage:

        my $fh = open_file($path, '<');

=head3 EXAMPLE

        my $out = open_file('/tmp/report.bin', '>');
        print {$out} $bytes;
        close $out;

=head3 API SPECIFICATION

=head4 INPUT

        [
                { name => 'path', type => 'string', min => 1 },
                { name => 'mode', type => 'string', memberof => [ '<', '>', '>>' ] },
        ]

=head4 OUTPUT

        { type => 'handle' }

=head3 MESSAGES

        Key        Meaning                          Resolution
        io.read    the file cannot be opened        check it exists and is readable
        io.write   the file cannot be created       check the directory is writable
        io.append  the file cannot be appended to   check permissions and free space

=head3 FORMAL SPECIFICATION

        open_file : PATH × MODE ↛ HANDLE
        pre  mode ∈ {<, >, >>} ∧ (mode = < ⇒ path ∈ dom FS)
        post mode = > ⇒ FS'(path) = ⟨⟩

=cut
121
122sub open_file {
123
3487
4589
        my ($path, $mode) = @_;
124
125
3487
2517
        my $fh;
126
3487
3484
5770
385230
        eval { open $fh, $mode, $path; 1 }
127
3487
2539
                or throw($OPEN_KEY{$mode} || 'io.read', { path => $path, error => os_error($@) });
128
3484
6060
        binmode $fh;
129
130
3484
166354
        return $fh;
131}
132
133 - 172
=head2 read_text

Purpose: read a whole file as bytes.

Args: C<$path>.

Returns: the contents (an empty string for an empty file).

Side Effects: none.

Usage:

        my $json = read_text($config_path);

=head3 EXAMPLE

        my $marker = read_text("$root/.filesync-root");

=head3 API SPECIFICATION

=head4 INPUT

        [ { name => 'path', type => 'string', min => 1 } ]

=head4 OUTPUT

        { type => 'string' }

=head3 MESSAGES

        Key       Meaning                     Resolution
        io.read   the file cannot be opened   check it exists and is readable

=head3 FORMAL SPECIFICATION

        read_text : PATH ↛ seq BYTE
        pre  path ∈ dom FS
        post result = FS(path)

=cut
173
174sub read_text {
175
1338
211908
        my ($path) = @_;
176
177
1338
1896
        my $fh = open_file($path, '<');
178
1335
1335
1335
1157
3012
17290
        my $text = do { local $/; <$fh> };
179
1335
2494
        close $fh;
180
181
1335
122659
        return defined $text ? $text : '';
182}
183
184 - 230
=head2 temp_file

Purpose: create a temporary file beside its final location.

Args: C<$directory>; created if missing.

Returns: an arrayref C<[ $filehandle, $path ]>.

Side Effects: creates the file exclusively (C<O_EXCL>), so a planted file or
symlink of the same name cannot be written through.  Its permissions are those
an ordinary C<open> would give under the current umask.  The name starts with
C<.filesync-tmp->, which the scanner ignores and stale-temp cleanup removes.

Usage:

        my ($fh, $temp) = @{ temp_file($directory) };

=head3 EXAMPLE

        my ($fh, $temp) = @{ temp_file('/home/foo/Documents/notes') };
        print {$fh} $bytes;
        close $fh;
        install_file($temp, '/home/foo/Documents/notes/plan.txt');

=head3 API SPECIFICATION

=head4 INPUT

        [ { name => 'directory', type => 'string', min => 1 } ]

=head4 OUTPUT

        { type => 'arrayref', min => 2, max => 2 }

=head3 MESSAGES

        Key          Meaning                                  Resolution
        io.mkdir     the directory cannot be created          check permissions
        io.temp      no temporary file can be made there      check permissions and free space

=head3 FORMAL SPECIFICATION

        Î”FS
        dir? : PATH ; t! : PATH
        t! ∉ dom FS ∧ dirname(t!) = dir? ∧ FS' = FS ∪ {t! ↦ ⟨⟩}

=cut
231
232sub temp_file {
233
1018
1242
        my ($directory) = @_;
234
235
1018
1442
        ensure_dir($directory);
236
237
1018
734
        my ($fh, $path);
238
1018
861
        eval {
239
1018
2677
                ($fh, $path) = File::Temp::tempfile($TEMP_TEMPLATE, DIR => $directory, UNLINK => 0);
240
1018
169096
                1;
241        } or throw('io.temp', { path => $directory, error => os_error($@) });
242
243
1018
1945
        binmode $fh;
244
1018
35273
        chmod($CREATE_MODE & ~umask, $path) if $HAS_MODES;
245
246
1018
137096
        return [ $fh, $path ];
247}
248
249 - 297
=head2 write_atomic

Purpose: replace a file's contents so readers see the old or the new version,
never a mixture.

Args: C<$path>, C<$data> (bytes).

Returns: 1.

Side Effects: creates missing parent directories; writes a temporary file in
the same directory and renames it over C<$path>.  The temporary file is
removed if any step fails.

Usage:

        write_atomic($path, json_encode($config) . "\n");

=head3 EXAMPLE

        write_atomic('/tmp/state.json', qq({"ok":1}\n));

=head3 API SPECIFICATION

=head4 INPUT

        [
                { name => 'path', type => 'string', min => 1 },
                { name => 'data', type => 'string' },
        ]

=head4 OUTPUT

        { type => 'integer', min => 1, max => 1 }

=head3 MESSAGES

        Key        Meaning                                Resolution
        io.temp    no temporary file can be created       check permissions and space
        io.write   writing or closing the file failed     check free space
        io.rename  the new file cannot replace the old    check permissions

=head3 FORMAL SPECIFICATION

        Î”FS
        path? : PATH ; data? : seq BYTE
        FS'(path?) = data?
        âˆ€ t : dom FS' \ dom FS • ¬ prefix(TEMP, basename t)

=cut
298
299sub write_atomic {
300
753
287974
        my ($path, $data) = @_;
301
302
753
753
820
16647
        my ($fh, $temp) = @{ temp_file(dirname($path)) };
303
304        # print is outside autodie's reach, so its failure is checked here.
305
753
753
965
3396
        my $written = print {$fh} $data;
306
753
753
753
764
1248
31383
        my $closed  = eval { close $fh; 1 };
307
753
2328
        unless ($written && $closed) {
308
0
0
                my $error = os_error($@);
309
0
0
                _discard($temp);
310
0
0
                throw('io.write', { path => $temp, error => $error });
311        }
312
313
753
1596
        _rename_or_discard($temp, $path);
314
753
1992
        return 1;
315}
316
317 - 370
=head2 install_file

Purpose: move a verified temporary file into place.

Args: C<$temp>, C<$target>, optional C<$mtime> (epoch, may be fractional) and
C<$mode> (permission bits).

Returns: C<$target>.

Side Effects: sets the mode (not on Windows) and mtime on C<$temp>, removes
whatever is at C<$target> (a file, a symlink or an empty directory), then
renames.  C<$temp> is removed if the rename fails.

Usage:

        install_file($temp, $abs, $mtime, $mode);

=head3 EXAMPLE

        my ($fh, $temp) = @{ temp_file($dir) };
        print {$fh} $content;
        close $fh;
        install_file($temp, "$dir/report.txt", 1_700_000_000, oct(644));

=head3 API SPECIFICATION

=head4 INPUT

        [
                { name => 'temp',   type => 'string', min => 1 },
                { name => 'target', type => 'string', min => 1 },
                { name => 'mtime',  type => 'number', optional => 1 },
                { name => 'mode',   type => 'integer', optional => 1, min => 0 },
        ]

=head4 OUTPUT

        { type => 'string', min => 1 }

=head3 MESSAGES

        Key                Meaning                                     Resolution
        io.remove          the old file cannot be removed              check permissions
        io.replace_dir     a non-empty directory is in the way         move it aside
        io.rename          the new file cannot be moved into place     check permissions

=head3 FORMAL SPECIFICATION

        Î”FS
        temp?, target? : PATH
        pre  temp? ∈ dom FS
        post FS'(target?) = FS(temp?) ∧ temp? ∉ dom FS'

=cut
371
372sub install_file {
373
263
351
        my ($temp, $target, $mtime, $mode) = @_;
374
375
263
736
        chmod($mode & $MODE_MASK, $temp) if defined $mode && $HAS_MODES;
376
263
9195
        set_mtime($temp, $mtime);
377
378
263
369
        clear_target($target);
379
263
494
        _rename_or_discard($temp, $target);
380
381
263
346
        return $target;
382}
383
384 - 425
=head2 clear_target

Purpose: make room for a file at C<$path>.

Args: C<$path>.

Returns: 1.

Side Effects: removes a symlink or file at C<$path>, or an empty directory.
A non-empty directory is never removed.

Usage:

        clear_target($abs);

=head3 EXAMPLE

        clear_target('/tmp/old-copy.txt');

=head3 API SPECIFICATION

=head4 INPUT

        [ { name => 'path', type => 'string', min => 1 } ]

=head4 OUTPUT

        { type => 'integer', min => 1, max => 1 }

=head3 MESSAGES

        Key              Meaning                              Resolution
        io.remove        the file or symlink cannot go        check permissions
        io.replace_dir   a non-empty directory is in the way  move it aside

=head3 FORMAL SPECIFICATION

        Î”FS
        path? : PATH
        path? ∈ dom FS ∧ (kind path? ≠ dir ∨ children path? = ∅) ⇒ path? ∉ dom FS'

=cut
426
427sub clear_target {
428
263
304
        my ($path) = @_;
429
430        # Symlinks first: -d follows them, and a link to a directory must be
431        # unlinked, not rmdir'ed.
432
263
1995
        if (-l $path || (-e _ && !-d _)) {
433
13
13
13
21
48
2193
                eval { unlink $path; 1 } or throw('io.remove', { path => $path, error => os_error($@) });
434        } elsif (-d _) {
435
0
0
0
0
0
0
                eval { rmdir $path; 1 } or throw('io.replace_dir', { path => $path });
436        }
437
438
263
228
        return 1;
439}
440
441 - 487
=head2 set_mtime

Purpose: set a file's modification time, keeping sub-second precision where
the platform allows.

Args: C<$path>, C<$mtime> (epoch; C<undef> leaves the file alone).

Returns: 1.

Side Effects: changes the file's atime and mtime.  Where C<Time::HiRes::utime>
is unimplemented (some Windows builds) whole seconds are stored, which the
2-second comparison tolerance already allows for.

Usage:

        set_mtime($temp, 1_700_000_123.456);

=head3 EXAMPLE

        set_mtime('/tmp/a.txt', time - 3600);

=head3 API SPECIFICATION

=head4 INPUT

        [
                { name => 'path',  type => 'string', min => 1 },
                { name => 'mtime', type => 'number', optional => 1 },
        ]

=head4 OUTPUT

        { type => 'integer', min => 1, max => 1 }

=head3 MESSAGES

        Key        Meaning                         Resolution
        io.utime   the timestamp cannot be set     check ownership of the file

=head3 FORMAL SPECIFICATION

        Î”FS
        path? : PATH ; m? : ℝ ∪ {⊥}
        m? = ⊥ ⇒ FS' = FS
        m? ≠ ⊥ ⇒ |mtime'(path?) − m?| < 1

=cut
488
489sub set_mtime {
490
266
77957
        my ($path, $mtime) = @_;
491
492
266
266
264
530
1669
456
        return 1 if !defined $mtime || eval { Time::HiRes::utime($mtime, $mtime, $path); 1 };
493
494        # Only the "unimplemented" croak is expected; anything else is a real
495        # failure, reported with the reason Time::HiRes gave.
496
2
11
        my $error = $@;
497
2
11
        throw('io.utime', { path => $path, error => strip_location($error) })
498                unless !ref $error && $error =~ /unimplemented/i;
499
500
1
1
1
2
3
241
        eval { utime(int($mtime), int($mtime), $path); 1 }
501                or throw('io.utime', { path => $path, error => os_error($@) });
502
1
3
        return 1;
503}
504
505 - 543
=head2 sha256_file

Purpose: hash a file's contents.

Args: C<$path>.

Returns: the lower-case hex SHA-256, or C<undef> when C<$path> is not a
readable regular file.

Side Effects: none.

Usage:

        my $hash = sha256_file($abs);

=head3 EXAMPLE

        print sha256_file('/etc/hostname'), "\n";

=head3 API SPECIFICATION

=head4 INPUT

        [ { name => 'path', type => 'string', min => 1 } ]

=head4 OUTPUT

        { type => 'string', optional => 1, matches => qr/\A[0-9a-f]{64}\z/ }

=head3 MESSAGES

None; an unreadable file yields C<undef>.

=head3 FORMAL SPECIFICATION

        sha256_file : PATH ⇸ HEX64
        sha256_file(p) = sha256(FS(p))   if p ∈ dom FS ∧ kind p = file ∧ readable p

=cut
544
545sub sha256_file {
546
6
8
        my ($path) = @_;
547
548
6
42
        my $sha = Digest::SHA->new(256);
549
6
86
        my $ok = -f $path && eval { $sha->addfile($path, 'b'); 1 };
550
551
6
52
        return $ok ? $sha->hexdigest() : undef;
552}
553
554 - 592
=head2 list_dir

Purpose: the entries of a directory.

Args: C<$directory>.

Returns: an arrayref of names, sorted, without C<.> and C<..>; C<undef> when
the directory cannot be read (C<$!> says why).

Side Effects: none.

Usage:

        my $names = list_dir($dir) or return 0;

=head3 EXAMPLE

        print "$_\n" for @{ list_dir('/tmp') || [] };

=head3 API SPECIFICATION

=head4 INPUT

        [ { name => 'directory', type => 'string', min => 1 } ]

=head4 OUTPUT

        { type => 'arrayref', optional => 1 }

=head3 MESSAGES

None; an unreadable directory yields C<undef>.

=head3 FORMAL SPECIFICATION

        list_dir : PATH ⇸ seq NAME
        list_dir(d) = sort(children d \ {".", ".."})   if readable d

=cut
593
594sub list_dir {
595
1473
1647
        my ($directory) = @_;
596
597        # A missing or unreadable directory is an answer here, not an exception,
598        # so autodie is switched off for this one call and $! is left for the
599        # caller to report.
600
1473
1095
        my $dh;
601
327
327
327
1473
1473
159909
281
1111
1066
18381
        my $opened = do { no autodie qw(opendir); opendir($dh, $directory) };
602
1473
2209
        return undef unless $opened;    ## no critic (Subroutines::ProhibitExplicitReturnUndef)
603
604
1472
6375
14788
9664
        my @names = sort grep { $_ ne '.' && $_ ne '..' } readdir($dh);
605
1472
3148
        closedir $dh;
606
607
1472
96750
        return \@names;
608}
609
610 - 655
=head2 copy_preserving

Purpose: copy one regular file, keeping its mode and mtime.

Args: C<$source>, C<$destination>, optional C<\@stat> of the source (saves a
second C<stat> when the caller already has one).

Returns: 1.

Side Effects: creates C<$destination>'s parent directories; overwrites
C<$destination>.

Usage:

        copy_preserving($from, $to, \@stat);

=head3 EXAMPLE

        copy_preserving('/srv/a.txt', '/backup/a.txt');

=head3 API SPECIFICATION

=head4 INPUT

        [
                { name => 'source',      type => 'string', min => 1 },
                { name => 'destination', type => 'string', min => 1 },
                { name => 'stat',        type => 'arrayref', optional => 1 },
        ]

=head4 OUTPUT

        { type => 'integer', min => 1, max => 1 }

=head3 MESSAGES

        Key       Meaning                      Resolution
        io.copy   the copy could not be made   check space and permissions

=head3 FORMAL SPECIFICATION

        Î”FS
        src?, dst? : PATH
        FS'(dst?) = FS(src?) ∧ mtime'(dst?) = mtime(src?) ∧ mode'(dst?) = mode(src?)

=cut
656
657sub copy_preserving {
658
16
22
        my ($source, $destination, $stat) = @_;
659
660
16
16
46
23
        $stat = [ Time::HiRes::stat($source) ] unless ref($stat) eq 'ARRAY' && @{$stat};
661
16
409
        ensure_dir(dirname($destination));
662
663
16
47
        File::Copy::copy($source, $destination)
664                or throw('io.copy', { from => $source, to => $destination, error => "$!" });
665
666
16
2149
        chmod($stat->[2] & $MODE_MASK, $destination) if $HAS_MODES;
667
16
1082
        utime($stat->[9], $stat->[9], $destination);
668
669
16
1290
        return 1;
670}
671
672 - 718
=head2 append_line

Purpose: append one line to a log or journal.

Args: C<$path>, C<$line> (without a newline).

Returns: 1.

Side Effects: creates C<$path> and its directory if needed.  If the file ends
in a torn line from a killed run, a newline is written first so the new entry
is not glued to the fragment.

Usage:

        append_line($journal, json_encode($event));

=head3 EXAMPLE

        append_line('/tmp/run.log', 'started');

=head3 API SPECIFICATION

=head4 INPUT

        [
                { name => 'path', type => 'string', min => 1 },
                { name => 'line', type => 'string' },
        ]

=head4 OUTPUT

        { type => 'integer', min => 1, max => 1 }

=head3 MESSAGES

        Key         Meaning                           Resolution
        io.append   the file cannot be appended to    check permissions and space

=head3 FORMAL SPECIFICATION

        Î”FS
        path? : PATH ; line? : seq CHAR
        let old = FS(path?) if path? ∈ dom FS, else ⟨⟩
        let sep = "\n" if old ≠ ⟨⟩ ∧ last old ≠ "\n", else ⟨⟩
        FS'(path?) = old ⁀ sep ⁀ line? ⁀ "\n"

=cut
719
720sub append_line {
721
711
125801
        my ($path, $line) = @_;
722
723
711
18360
        ensure_dir(dirname($path));
724
711
1784
        my $prefix = _ends_mid_line($path) ? "\n" : '';
725
726
711
1069
        my $fh = open_file($path, '>>');
727
711
711
696
2234
        my $written = print {$fh} $prefix, $line, "\n";
728
711
711
711
705
1028
19475
        my $closed  = eval { close $fh; 1 };
729
711
2038
        throw('io.append', { path => $path, error => os_error($@) }) unless $written && $closed;
730
731
711
1391
        return 1;
732}
733
734 - 773
=head2 ensure_dir

Purpose: make sure a directory exists.

Args: C<$directory>.

Returns: C<$directory>.

Side Effects: creates it and any missing parents.

Usage:

        ensure_dir(dirname($path));

=head3 EXAMPLE

        ensure_dir('/tmp/a/b/c');

=head3 API SPECIFICATION

=head4 INPUT

        [ { name => 'directory', type => 'string', min => 1 } ]

=head4 OUTPUT

        { type => 'string', min => 1 }

=head3 MESSAGES

        Key        Meaning                          Resolution
        io.mkdir   the directory cannot be created  check permissions

=head3 FORMAL SPECIFICATION

        Î”FS
        d? : PATH
        kind'(d?) = dir ∧ ∀ a : ancestors d? • kind'(a) = dir

=cut
774
775sub ensure_dir {
776
3058
3672
        my ($directory) = @_;
777
778
3058
18071
        unless (-d $directory) {
779
653
66287
                make_path($directory, { error => \my $errors });
780
653
3288
                throw('io.mkdir', { path => $directory, error => _path_error($errors) })
781                        unless -d $directory;
782        }
783
784
3058
3031
        return $directory;
785}
786
787# _rename_or_discard
788# Purpose:     rename a temporary file over its target.
789# Entry:       $temp exists; $target's directory exists.
790# Exit:        returns 1, or raises io.rename after removing $temp.
791# Side effects: $temp disappears either way.
792sub _rename_or_discard :Private {
793        my ($temp, $target) = @_;
794
795        unless (eval { rename $temp, $target; 1 }) {
796                my $error = os_error($@);
797                _discard($temp);
798                throw('io.rename', { from => $temp, to => $target, error => $error });
799        }
800
801        return 1;
802
327
327
327
121815
292
5834
}
803
804# _discard
805# Purpose:     best-effort removal of a temporary file after a failure.
806# Entry:       $path may or may not exist.
807# Exit:        returns 1.
808# Side effects: removes $path when it can; never raises, so the original
809#              error is the one the caller reports.
810sub _discard :Private {
811        my ($path) = @_;
812
813
327
327
327
93536
326
673
        { no autodie qw(unlink); unlink $path }
814        return 1;
815
327
327
327
35888
288
2849
}
816
817# _ends_mid_line
818# Purpose:     detect a log or journal whose last line was torn by a kill.
819# Entry:       $path may or may not exist.
820# Exit:        returns 1 when the file is non-empty and its last byte is not
821#              a newline, otherwise 0.
822# Side effects: none.
823sub _ends_mid_line :Private {
824        my ($path) = @_;
825
826        return 0 unless -s $path;
827
828        my $fh = open_file($path, '<');
829        seek($fh, -1, 2);
830        read($fh, my $last, 1);
831        close $fh;
832
833        return (defined $last && $last ne "\n") ? 1 : 0;
834
327
327
327
100870
268
2646
}
835
836# _path_error
837# Purpose:     the first reason File::Path recorded, for an io.mkdir message.
838# Entry:       $errors is File::Path's error arrayref (may be undef or empty).
839# Exit:        returns the reason text, or the current $! when there is none.
840# Side effects: none.
841sub _path_error :Private {
842        my ($errors) = @_;
843
844        my ($first) = @{ $errors || [] };
845        my ($reason) = $first ? values %{$first} : ();
846        return defined $reason && length $reason ? $reason : "$!";
847
327
327
327
93015
326
2494
}
848
8491;
850