File Coverage

File:blib/lib/App/FileSync/Verify.pm
Coverage:97.9%

linestmtbrancondsubtimecode
1package App::FileSync::Verify;
2
3# Compare a tree with the record of what it should hold (the server index, or
4# the client's cached base) and report the differences.  Read-only.
5
6
314
314
314
753
258
4325
use App::FileSync::Platform;
7
8
314
314
314
509
226
2431
use strict;
9
314
314
314
440
214
6457
use warnings;
10
314
314
314
481
235
699
use autodie qw(:all);
11
12
314
314
314
667611
302
4974
use File::Spec;
13
314
314
314
591
3470
11088
use List::Util 1.54 qw(sample);
14
314
314
314
656
275
1061
use Sub::Private { mode => 'enforce' };
15
16
314
314
314
32713
270
6147
use App::FileSync::Args qw(named);
17
314
314
314
611
252
13653
use App::FileSync::Constants qw($TYPE_FILE $TYPE_DIR);
18
314
314
314
637
229
5854
use App::FileSync::I18N qw(message throw);
19
314
314
314
587
294
4697
use App::FileSync::IO qw(sha256_file);
20
314
314
314
575
239
2720
use App::FileSync::Scan;
21
22
314
314
314
427
248
602
use version; our $VERSION = version->declare('0.001.1');
23
24# The type of a manifest entry; older entries without "t" are files.  Lexical
25# because it runs several times per path.
26my $type_of = sub {
27        my ($entry) = @_;
28        return (ref($entry) eq 'HASH' && defined $entry->{t}) ? $entry->{t} : $TYPE_FILE;
29};
30
31=encoding utf8
32
33 - 125
=head1 NAME

App::FileSync::Verify - check a tree against its recorded manifest

=head1 SYNOPSIS

        my $report = App::FileSync::Verify::verify(
                root => $master, entries => $index, ignore => $ignore, sample => 50,
        );
        print $report->{ok} ? "clean\n" : "drift\n";

=head1 FUNCTIONS

=head2 verify

Purpose: report where a tree no longer matches its record.

Args: C<root> (required, a directory), C<entries> (the recorded manifest),
C<ignore> (an L<App::FileSync::Ignore>), C<deep> (rehash every file),
C<sample> (without C<deep>, rehash this many unchanged-looking files chosen at
random).

Without C<deep>, entries whose size and mtime still match are trusted, which
is what makes a verify of a large tree cheap.

Returns: a hashref:

        root, ok (1 when nothing differs), deep, files (regular files found),
        known (entries in the record), rehashed, trusted, sampled,
        modified => [ { p, expected, got } | { p, reason } | { p, expected, got, sampled } ],
        missing  => [ { p, t } ],
        extra    => [ { p, t } ]

Each list is sorted by path.

Side Effects: reads files; never writes.

Usage:

        my $report = App::FileSync::Verify::verify(root => $root, entries => $base, deep => 1);

=head3 EXAMPLE

        my $report = App::FileSync::Verify::verify(root => '/srv/filesync/master/docs', entries => $index);
        print "missing: $_->{p}\n" for @{ $report->{missing} };

=head3 API SPECIFICATION

=head4 INPUT

        {
                root    => { type => 'string', min => 1 },
                entries => { type => 'hashref', optional => 1 },
                ignore  => { type => 'object', optional => 1 },
                deep    => { type => 'boolean', optional => 1 },
                sample  => { type => 'integer', min => 0, optional => 1 },
        }

=head4 OUTPUT

        {
                type => 'hashref',
                schema => {
                        ok       => { type => 'integer', min => 0, max => 1 },
                        modified => { type => 'arrayref' },
                        missing  => { type => 'arrayref' },
                        extra    => { type => 'arrayref' },
                },
        }

=head3 MESSAGES

        Key                    Meaning                        Resolution
        verify.no_root         root is not a directory        check the path
        verify.type_changed    (a reason in "modified")       a file became a directory or back

=head3 FORMAL SPECIFICATION

        let M = scan(root).manifest ; E = entries
        missing  = { p ∈ dom E | p ∉ dom M ∧ ¬(E(p).t = d ∧ isdir(root/p)) }
        modified = { p ∈ dom E ∩ dom M | E(p).t ≠ M(p).t ∨ (E(p).t = f ∧ E(p).h ≠ M(p).h) }
        extra    = dom M \ dom E \ (missing ∪ modified)
        ok ⇔ missing = modified = extra = ∅

=head3 PSEUDOCODE

        scan the tree, reusing recorded hashes unless deep
        for each recorded path: missing, type changed, or hash differs
        every scanned path not recorded is extra
        if sampling: rehash that many trusted files and add any that differ
        sort the three lists and count

=cut
126
127sub verify {
128
25
238
        my $args = named(\@_, {
129                root    => { type => 'string', min => 1 },
130                entries => { type => 'hashref', optional => 1 },
131                ignore  => { type => 'object', optional => 1 },
132                deep    => { type => 'boolean', optional => 1 },
133                sample  => { type => 'integer', min => 0, optional => 1 },
134        }, 'Verify::verify');
135
136
25
54
        my $root = $args->{root};
137
25
156
        throw('verify.no_root', { root => $root }) unless -d $root;
138
139
25
40
        my $entries = $args->{entries} || {};
140
25
38
        my $deep    = $args->{deep} ? 1 : 0;
141
142        my $scan = App::FileSync::Scan::scan(
143                root   => $root,
144                ignore => $args->{ignore},
145
25
92
                cache  => $deep ? {} : $entries,
146        );
147
25
28
        my $manifest = $scan->{manifest};
148
149
25
48
        my $found = _compare($root, $entries, $manifest);
150        my $sampled = ($deep || !$args->{sample})
151                ? 0
152
25
77
                : _sample_check($root, $entries, $manifest, $args->{sample}, $found);
153
154        my %sorted = map {
155
25
75
39
64
                my $list = $found->{$_};
156
75
7
75
49
14
125
                $_ => [ sort { $a->{p} cmp $b->{p} } @{$list} ];
157        } qw(modified missing extra);
158
159        return {
160                root     => $root,
161                ok       => (@{ $sorted{modified} } || @{ $sorted{missing} } || @{ $sorted{extra} }) ? 0 : 1,
162                deep     => $deep,
163
50
25
84
38
                files    => scalar(grep { $type_of->($manifest->{$_}) eq $TYPE_FILE } keys %{$manifest}),
164
25
301
                known    => scalar(keys %{$entries}),
165                rehashed => $scan->{stats}{hashed} + $sampled,
166                trusted  => $scan->{stats}{cached},
167
25
31
                sampled  => $sampled,
168                %sorted,
169        };
170}
171
172# _compare
173# Purpose:     compare the record with what the scan found.
174# Entry:       $entries is the record; $manifest the fresh scan.
175# Exit:        returns { modified, missing, extra, seen } (lists unsorted;
176#              seen is the set of paths already reported).
177# Side effects: none; stats one directory per recorded empty dir that is gone
178#              from the manifest.
179sub _compare :Private {
180        my ($root, $entries, $manifest) = @_;
181
182        my (%found, %seen);
183        @found{qw(modified missing extra)} = ([], [], []);
184
185        for my $path (sort keys %{$entries}) {
186                my $entry = $entries->{$path};
187                next unless ref($entry) eq 'HASH';
188
189                my $type = $type_of->($entry);
190                my $have = $manifest->{$path};
191                my $problem;
192
193                if (!$have) {
194                        # The scanner only reports directories that are empty, so an empty
195                        # directory that has since gained a file is not missing.
196                        next if $type eq $TYPE_DIR && -d File::Spec->catfile($root, split m{/}, $path);
197                        push @{ $found{missing} }, { p => $path, t => $type };
198                } elsif ($type ne $type_of->($have)) {
199                        push @{ $found{modified} }, { p => $path, reason => message('verify.type_changed') };
200                } elsif ($type eq $TYPE_FILE && defined $entry->{h} && defined $have->{h} && $entry->{h} ne $have->{h}) {
201                        push @{ $found{modified} }, { p => $path, expected => $entry->{h}, got => $have->{h} };
202                } else {
203                        next;
204                }
205                $seen{$path} = 1;
206        }
207
208        for my $path (sort keys %{$manifest}) {
209                next if $entries->{$path} || $seen{$path};
210                push @{ $found{extra} }, { p => $path, t => $type_of->($manifest->{$path}) };
211        }
212
213        $found{seen} = \%seen;
214        return \%found;
215
314
314
314
135176
306
3974
}
216
217# _sample_check
218# Purpose:     rehash a random sample of files the scan trusted.
219# Entry:       $wanted > 0; $found is _compare's result, updated in place.
220# Exit:        returns how many files were rehashed.
221# Side effects: reads the sampled files; appends to $found->{modified}.
222sub _sample_check :Private {
223        my ($root, $entries, $manifest, $wanted, $found) = @_;
224
225        # Only a file on both sides can be compared by content.
226        my @candidates = grep {
227                ref($entries->{$_}) eq 'HASH'
228                        && $type_of->($entries->{$_}) eq $TYPE_FILE
229                        && $manifest->{$_}
230                        && $type_of->($manifest->{$_}) eq $TYPE_FILE
231        } sort keys %{$entries};
232
233        my $sampled = 0;
234        for my $path (sample($wanted, @candidates)) {
235                my $got = sha256_file(File::Spec->catfile($root, split m{/}, $path));
236                next unless defined $got;
237                $sampled++;
238
239                my $expected = $entries->{$path}{h};
240                next if !defined $expected || $got eq $expected || $found->{seen}{$path};
241
242                push @{ $found->{modified} }, { p => $path, expected => $expected, got => $got, sampled => 1 };
243                $found->{seen}{$path} = 1;
244        }
245
246        return $sampled;
247
314
314
314
109594
288
2711
}
248
2491;
250