| File: | blib/lib/App/FileSync/Verify.pm |
| Coverage: | 97.9% |
| line | stmt | bran | cond | sub | time | code |
|---|---|---|---|---|---|---|
| 1 | package App::FileSync::Verify; | |||||
| 2 | ||||||
| 3 | # Compare a tree with the record of what it should hold (the server index, or | |||||
| 4 | # the client's cached base) and report the differences. Read-only. | |||||
| 5 | ||||||
| 6 | 314 314 314 | 753 258 4325 | use App::FileSync::Platform; | |||
| 7 | ||||||
| 8 | 314 314 314 | 509 226 2431 | use strict; | |||
| 9 | 314 314 314 | 440 214 6457 | use warnings; | |||
| 10 | 314 314 314 | 481 235 699 | use autodie qw(:all); | |||
| 11 | ||||||
| 12 | 314 314 314 | 667611 302 4974 | use File::Spec; | |||
| 13 | 314 314 314 | 591 3470 11088 | use List::Util 1.54 qw(sample); | |||
| 14 | 314 314 314 | 656 275 1061 | use Sub::Private { mode => 'enforce' }; | |||
| 15 | ||||||
| 16 | 314 314 314 | 32713 270 6147 | use App::FileSync::Args qw(named); | |||
| 17 | 314 314 314 | 611 252 13653 | use App::FileSync::Constants qw($TYPE_FILE $TYPE_DIR); | |||
| 18 | 314 314 314 | 637 229 5854 | use App::FileSync::I18N qw(message throw); | |||
| 19 | 314 314 314 | 587 294 4697 | use App::FileSync::IO qw(sha256_file); | |||
| 20 | 314 314 314 | 575 239 2720 | use App::FileSync::Scan; | |||
| 21 | ||||||
| 22 | 314 314 314 | 427 248 602 | use version; our $VERSION = version->declare('0.001.1'); | |||
| 23 | ||||||
| 24 | # The type of a manifest entry; older entries without "t" are files. Lexical | |||||
| 25 | # because it runs several times per path. | |||||
| 26 | my $type_of = sub { | |||||
| 27 | my ($entry) = @_; | |||||
| 28 | return (ref($entry) eq 'HASH' && defined $entry->{t}) ? $entry->{t} : $TYPE_FILE; | |||||
| 29 | }; | |||||
| 30 | ||||||
| 31 | =encoding utf8 | |||||
| 32 | ||||||
| 33 - 125 | =head1 NAME
App::FileSync::Verify - check a tree against its recorded manifest
=head1 SYNOPSIS
my $report = App::FileSync::Verify::verify(
root => $master, entries => $index, ignore => $ignore, sample => 50,
);
print $report->{ok} ? "clean\n" : "drift\n";
=head1 FUNCTIONS
=head2 verify
Purpose: report where a tree no longer matches its record.
Args: C<root> (required, a directory), C<entries> (the recorded manifest),
C<ignore> (an L<App::FileSync::Ignore>), C<deep> (rehash every file),
C<sample> (without C<deep>, rehash this many unchanged-looking files chosen at
random).
Without C<deep>, entries whose size and mtime still match are trusted, which
is what makes a verify of a large tree cheap.
Returns: a hashref:
root, ok (1 when nothing differs), deep, files (regular files found),
known (entries in the record), rehashed, trusted, sampled,
modified => [ { p, expected, got } | { p, reason } | { p, expected, got, sampled } ],
missing => [ { p, t } ],
extra => [ { p, t } ]
Each list is sorted by path.
Side Effects: reads files; never writes.
Usage:
my $report = App::FileSync::Verify::verify(root => $root, entries => $base, deep => 1);
=head3 EXAMPLE
my $report = App::FileSync::Verify::verify(root => '/srv/filesync/master/docs', entries => $index);
print "missing: $_->{p}\n" for @{ $report->{missing} };
=head3 API SPECIFICATION
=head4 INPUT
{
root => { type => 'string', min => 1 },
entries => { type => 'hashref', optional => 1 },
ignore => { type => 'object', optional => 1 },
deep => { type => 'boolean', optional => 1 },
sample => { type => 'integer', min => 0, optional => 1 },
}
=head4 OUTPUT
{
type => 'hashref',
schema => {
ok => { type => 'integer', min => 0, max => 1 },
modified => { type => 'arrayref' },
missing => { type => 'arrayref' },
extra => { type => 'arrayref' },
},
}
=head3 MESSAGES
Key Meaning Resolution
verify.no_root root is not a directory check the path
verify.type_changed (a reason in "modified") a file became a directory or back
=head3 FORMAL SPECIFICATION
let M = scan(root).manifest ; E = entries
missing = { p â dom E | p â dom M ⧠¬(E(p).t = d â§ isdir(root/p)) }
modified = { p â dom E â© dom M | E(p).t â M(p).t ⨠(E(p).t = f â§ E(p).h â M(p).h) }
extra = dom M \ dom E \ (missing ⪠modified)
ok â missing = modified = extra = â
=head3 PSEUDOCODE
scan the tree, reusing recorded hashes unless deep
for each recorded path: missing, type changed, or hash differs
every scanned path not recorded is extra
if sampling: rehash that many trusted files and add any that differ
sort the three lists and count
=cut | |||||
| 126 | ||||||
| 127 | sub verify { | |||||
| 128 | 25 | 238 | my $args = named(\@_, { | |||
| 129 | root => { type => 'string', min => 1 }, | |||||
| 130 | entries => { type => 'hashref', optional => 1 }, | |||||
| 131 | ignore => { type => 'object', optional => 1 }, | |||||
| 132 | deep => { type => 'boolean', optional => 1 }, | |||||
| 133 | sample => { type => 'integer', min => 0, optional => 1 }, | |||||
| 134 | }, 'Verify::verify'); | |||||
| 135 | ||||||
| 136 | 25 | 54 | my $root = $args->{root}; | |||
| 137 | 25 | 156 | throw('verify.no_root', { root => $root }) unless -d $root; | |||
| 138 | ||||||
| 139 | 25 | 40 | my $entries = $args->{entries} || {}; | |||
| 140 | 25 | 38 | my $deep = $args->{deep} ? 1 : 0; | |||
| 141 | ||||||
| 142 | my $scan = App::FileSync::Scan::scan( | |||||
| 143 | root => $root, | |||||
| 144 | ignore => $args->{ignore}, | |||||
| 145 | 25 | 92 | cache => $deep ? {} : $entries, | |||
| 146 | ); | |||||
| 147 | 25 | 28 | my $manifest = $scan->{manifest}; | |||
| 148 | ||||||
| 149 | 25 | 48 | my $found = _compare($root, $entries, $manifest); | |||
| 150 | my $sampled = ($deep || !$args->{sample}) | |||||
| 151 | ? 0 | |||||
| 152 | 25 | 77 | : _sample_check($root, $entries, $manifest, $args->{sample}, $found); | |||
| 153 | ||||||
| 154 | my %sorted = map { | |||||
| 155 | 25 75 | 39 64 | my $list = $found->{$_}; | |||
| 156 | 75 7 75 | 49 14 125 | $_ => [ sort { $a->{p} cmp $b->{p} } @{$list} ]; | |||
| 157 | } qw(modified missing extra); | |||||
| 158 | ||||||
| 159 | return { | |||||
| 160 | root => $root, | |||||
| 161 | ok => (@{ $sorted{modified} } || @{ $sorted{missing} } || @{ $sorted{extra} }) ? 0 : 1, | |||||
| 162 | deep => $deep, | |||||
| 163 | 50 25 | 84 38 | files => scalar(grep { $type_of->($manifest->{$_}) eq $TYPE_FILE } keys %{$manifest}), | |||
| 164 | 25 | 301 | known => scalar(keys %{$entries}), | |||
| 165 | rehashed => $scan->{stats}{hashed} + $sampled, | |||||
| 166 | trusted => $scan->{stats}{cached}, | |||||
| 167 | 25 | 31 | sampled => $sampled, | |||
| 168 | %sorted, | |||||
| 169 | }; | |||||
| 170 | } | |||||
| 171 | ||||||
| 172 | # _compare | |||||
| 173 | # Purpose: compare the record with what the scan found. | |||||
| 174 | # Entry: $entries is the record; $manifest the fresh scan. | |||||
| 175 | # Exit: returns { modified, missing, extra, seen } (lists unsorted; | |||||
| 176 | # seen is the set of paths already reported). | |||||
| 177 | # Side effects: none; stats one directory per recorded empty dir that is gone | |||||
| 178 | # from the manifest. | |||||
| 179 | sub _compare :Private { | |||||
| 180 | my ($root, $entries, $manifest) = @_; | |||||
| 181 | ||||||
| 182 | my (%found, %seen); | |||||
| 183 | @found{qw(modified missing extra)} = ([], [], []); | |||||
| 184 | ||||||
| 185 | for my $path (sort keys %{$entries}) { | |||||
| 186 | my $entry = $entries->{$path}; | |||||
| 187 | next unless ref($entry) eq 'HASH'; | |||||
| 188 | ||||||
| 189 | my $type = $type_of->($entry); | |||||
| 190 | my $have = $manifest->{$path}; | |||||
| 191 | my $problem; | |||||
| 192 | ||||||
| 193 | if (!$have) { | |||||
| 194 | # The scanner only reports directories that are empty, so an empty | |||||
| 195 | # directory that has since gained a file is not missing. | |||||
| 196 | next if $type eq $TYPE_DIR && -d File::Spec->catfile($root, split m{/}, $path); | |||||
| 197 | push @{ $found{missing} }, { p => $path, t => $type }; | |||||
| 198 | } elsif ($type ne $type_of->($have)) { | |||||
| 199 | push @{ $found{modified} }, { p => $path, reason => message('verify.type_changed') }; | |||||
| 200 | } elsif ($type eq $TYPE_FILE && defined $entry->{h} && defined $have->{h} && $entry->{h} ne $have->{h}) { | |||||
| 201 | push @{ $found{modified} }, { p => $path, expected => $entry->{h}, got => $have->{h} }; | |||||
| 202 | } else { | |||||
| 203 | next; | |||||
| 204 | } | |||||
| 205 | $seen{$path} = 1; | |||||
| 206 | } | |||||
| 207 | ||||||
| 208 | for my $path (sort keys %{$manifest}) { | |||||
| 209 | next if $entries->{$path} || $seen{$path}; | |||||
| 210 | push @{ $found{extra} }, { p => $path, t => $type_of->($manifest->{$path}) }; | |||||
| 211 | } | |||||
| 212 | ||||||
| 213 | $found{seen} = \%seen; | |||||
| 214 | return \%found; | |||||
| 215 | 314 314 314 | 135176 306 3974 | } | |||
| 216 | ||||||
| 217 | # _sample_check | |||||
| 218 | # Purpose: rehash a random sample of files the scan trusted. | |||||
| 219 | # Entry: $wanted > 0; $found is _compare's result, updated in place. | |||||
| 220 | # Exit: returns how many files were rehashed. | |||||
| 221 | # Side effects: reads the sampled files; appends to $found->{modified}. | |||||
| 222 | sub _sample_check :Private { | |||||
| 223 | my ($root, $entries, $manifest, $wanted, $found) = @_; | |||||
| 224 | ||||||
| 225 | # Only a file on both sides can be compared by content. | |||||
| 226 | my @candidates = grep { | |||||
| 227 | ref($entries->{$_}) eq 'HASH' | |||||
| 228 | && $type_of->($entries->{$_}) eq $TYPE_FILE | |||||
| 229 | && $manifest->{$_} | |||||
| 230 | && $type_of->($manifest->{$_}) eq $TYPE_FILE | |||||
| 231 | } sort keys %{$entries}; | |||||
| 232 | ||||||
| 233 | my $sampled = 0; | |||||
| 234 | for my $path (sample($wanted, @candidates)) { | |||||
| 235 | my $got = sha256_file(File::Spec->catfile($root, split m{/}, $path)); | |||||
| 236 | next unless defined $got; | |||||
| 237 | $sampled++; | |||||
| 238 | ||||||
| 239 | my $expected = $entries->{$path}{h}; | |||||
| 240 | next if !defined $expected || $got eq $expected || $found->{seen}{$path}; | |||||
| 241 | ||||||
| 242 | push @{ $found->{modified} }, { p => $path, expected => $expected, got => $got, sampled => 1 }; | |||||
| 243 | $found->{seen}{$path} = 1; | |||||
| 244 | } | |||||
| 245 | ||||||
| 246 | return $sampled; | |||||
| 247 | 314 314 314 | 109594 288 2711 | } | |||
| 248 | ||||||
| 249 | 1; | |||||
| 250 | ||||||