| File: | blib/lib/App/FileSync/Scan.pm |
| Coverage: | 88.9% |
| line | stmt | bran | cond | sub | time | code |
|---|---|---|---|---|---|---|
| 1 | package App::FileSync::Scan; | |||||
| 2 | ||||||
| 3 | # Walk a tree and describe it: one manifest entry per regular file (with its | |||||
| 4 | # SHA-256) and per empty directory. A previous manifest serves as a cache, so | |||||
| 5 | # a file whose size and mtime are unchanged is not read again. | |||||
| 6 | ||||||
| 7 | 315 315 315 | 892 235 4327 | use App::FileSync::Platform; | |||
| 8 | ||||||
| 9 | 315 315 315 | 495 193 2379 | use strict; | |||
| 10 | 315 315 315 | 425 220 6079 | use warnings; | |||
| 11 | 315 315 315 | 476 210 574 | use autodie qw(:all); | |||
| 12 | ||||||
| 13 | 315 315 315 | 664095 1349 3351 | use Digest::SHA (); | |||
| 14 | 315 315 315 | 530 243 3081 | use File::Spec; | |||
| 15 | 315 315 315 | 630 443 7711 | use Readonly; | |||
| 16 | 315 315 315 | 631 245 1056 | use Sub::Private { mode => 'enforce' }; | |||
| 17 | 315 315 315 | 33392 246 2857 | use Time::HiRes (); | |||
| 18 | ||||||
| 19 | 315 315 315 | 502 240 6057 | use App::FileSync::Args qw(named); | |||
| 20 | 315 315 315 | 646 266 15788 | use App::FileSync::Constants qw($MODE_MASK $TEMP_PREFIX $STALE_TEMP_SECONDS $TYPE_FILE $TYPE_DIR); | |||
| 21 | 315 315 315 | 613 245 5948 | use App::FileSync::I18N qw(message os_error); | |||
| 22 | 315 315 315 | 704 211 4320 | use App::FileSync::IO qw(list_dir); | |||
| 23 | ||||||
| 24 | 315 315 315 | 500 231 812 | use version; our $VERSION = version->declare('0.001.1'); | |||
| 25 | ||||||
| 26 | # A cached mtime within this much of the file's is the same mtime: JSON | |||||
| 27 | # round-trips the fractional part with float noise. | |||||
| 28 | Readonly my $MTIME_EPSILON => 0.001; | |||||
| 29 | ||||||
| 30 | # Matches a temporary file left by an interrupted run. | |||||
| 31 | Readonly my $TEMP_NAME => qr/\A\Q$TEMP_PREFIX\E/; | |||||
| 32 | ||||||
| 33 | # Describe one regular file, reusing the cached hash when size and mtime | |||||
| 34 | # match. Lexical rather than a named private sub because it runs once per | |||||
| 35 | # file in the tree; see App::FileSync/LIMITATIONS. | |||||
| 36 | # | |||||
| 37 | # Entry: $context is the scan state; $abs/$rel name the file; $stat is lstat. | |||||
| 38 | # Exit: returns the manifest entry, or undef after recording the file as | |||||
| 39 | # skipped (it could not be read). | |||||
| 40 | my $file_entry = sub { | |||||
| 41 | my ($context, $abs, $rel, $stat) = @_; | |||||
| 42 | ||||||
| 43 | my ($size, $mtime) = @{$stat}[7, 9]; | |||||
| 44 | my $cached = $context->{cache}{$rel}; | |||||
| 45 | ||||||
| 46 | my $hash; | |||||
| 47 | if ($cached | |||||
| 48 | && ($cached->{t} || '') eq $TYPE_FILE | |||||
| 49 | && defined $cached->{h} | |||||
| 50 | && defined $cached->{s} && $cached->{s} == $size | |||||
| 51 | && abs((defined $cached->{m} ? $cached->{m} : -1) - $mtime) < $MTIME_EPSILON) { | |||||
| 52 | $hash = $cached->{h}; | |||||
| 53 | $context->{stats}{cached}++; | |||||
| 54 | } else { | |||||
| 55 | my $sha = Digest::SHA->new(256); | |||||
| 56 | unless (eval { $sha->addfile($abs, 'b'); 1 }) { | |||||
| 57 | $context->{skipped}{$rel} = message('scan.unreadable', { error => $@ =~ s/\s+\z//r }); | |||||
| 58 | $context->{stats}{skipped}++; | |||||
| 59 | return undef; ## no critic (Subroutines::ProhibitExplicitReturnUndef) | |||||
| 60 | } | |||||
| 61 | $hash = $sha->hexdigest; | |||||
| 62 | $context->{stats}{hashed}++; | |||||
| 63 | } | |||||
| 64 | ||||||
| 65 | return { | |||||
| 66 | t => $TYPE_FILE, | |||||
| 67 | s => $size, | |||||
| 68 | m => $mtime, | |||||
| 69 | h => $hash, | |||||
| 70 | mode => $stat->[2] & $MODE_MASK, | |||||
| 71 | }; | |||||
| 72 | }; | |||||
| 73 | ||||||
| 74 | =encoding utf8 | |||||
| 75 | ||||||
| 76 - 164 | =head1 NAME
App::FileSync::Scan - describe a directory tree as a manifest
=head1 SYNOPSIS
my $scan = App::FileSync::Scan::scan(root => $root, ignore => $ignore, cache => $last);
for my $path (sort keys %{ $scan->{manifest} }) { ... }
printf "%d hashed, %d reused\n", @{ $scan->{stats} }{qw(hashed cached)};
=head1 FUNCTIONS
=head2 scan
Purpose: walk a tree and describe every regular file and empty directory.
Args: C<root> (required), C<ignore> (an L<App::FileSync::Ignore>), C<cache>
(a previous manifest whose entries are reused when size and mtime still match).
Returns: a hashref with three hashrefs:
manifest path => { t => 'f', s => size, m => mtime, h => sha256, mode => bits }
path => { t => 'd' } (only directories empty after ignores)
stats { files, dirs, bytes, hashed, cached, skipped }
skipped path => reason, for symlinks, special files and unreadable entries
Side Effects: reads every file that is not cached. Never writes.
Usage:
my $scan = App::FileSync::Scan::scan(root => $master, ignore => $ignore);
=head3 EXAMPLE
my $scan = App::FileSync::Scan::scan(root => '/home/foo/Documents');
print scalar(keys %{ $scan->{manifest} }), " entries\n";
=head3 API SPECIFICATION
=head4 INPUT
{
root => { type => 'string', min => 1 },
ignore => { type => 'object', can => 'match', optional => 1 },
cache => { type => 'hashref', optional => 1 },
}
=head4 OUTPUT
{
type => 'hashref',
schema => {
manifest => { type => 'hashref' },
stats => { type => 'hashref' },
skipped => { type => 'hashref' },
},
}
=head3 MESSAGES
Skip reasons (in C<skipped>, not raised):
Key Meaning
scan.unreadable a file could not be read
scan.cannot_stat an entry vanished or cannot be examined
scan.cannot_list a directory cannot be read
scan.symlink symbolic links are not synced
scan.special devices, sockets and fifos are not synced
Key Meaning Resolution
args.invalid no root was given pass root
=head3 FORMAL SPECIFICATION
Manifest == PATH ⸠ENTRY
scan(r, ign, c) = (M, S, K) where
dom M = { p | p â files(r) ⪠emptydirs(r) ⧠¬ ign(p) ⧠¬ unreadable(p) }
â p â dom M ⢠kind p = file â M(p).h = sha256(FS(r/p))
dom K â© dom M = â
=head3 PSEUDOCODE
for each entry under root, depth first, sorted:
skip ignored paths (and everything below an ignored directory)
directory: recurse; record it only if nothing inside was recorded
regular file: reuse the cached hash if size and mtime match, else hash it
anything else: record why it was skipped
=cut | |||||
| 165 | ||||||
| 166 | sub scan { | |||||
| 167 | 370 | 9431 | my $args = named(\@_, { | |||
| 168 | root => { type => 'string', min => 1 }, | |||||
| 169 | ignore => { type => 'object', optional => 1 }, | |||||
| 170 | cache => { type => 'hashref', optional => 1 }, | |||||
| 171 | }, 'Scan::scan'); | |||||
| 172 | ||||||
| 173 | my $context = { | |||||
| 174 | ignore => $args->{ignore}, | |||||
| 175 | cache => $args->{cache} || {}, | |||||
| 176 | manifest => {}, | |||||
| 177 | skipped => {}, | |||||
| 178 | 370 2220 | 1616 4130 | stats => { map { $_ => 0 } qw(files dirs bytes hashed cached skipped) }, | |||
| 179 | }; | |||||
| 180 | ||||||
| 181 | 370 | 1416 | _walk($context, $args->{root}, ''); | |||
| 182 | ||||||
| 183 | return { | |||||
| 184 | manifest => $context->{manifest}, | |||||
| 185 | stats => $context->{stats}, | |||||
| 186 | skipped => $context->{skipped}, | |||||
| 187 | 370 | 2098 | }; | |||
| 188 | } | |||||
| 189 | ||||||
| 190 - 234 | =head2 clean_stale_temps
Purpose: remove temporary files left behind by an interrupted run.
Args: C<root>, optional C<max_age> in seconds (default 3600).
Returns: the number of files removed.
Side Effects: deletes C<.filesync-tmp-*> files older than C<max_age> anywhere
under C<root>. Younger ones may belong to a run in progress and are left
alone. A file that cannot be removed is skipped.
Usage:
App::FileSync::Scan::clean_stale_temps(root => $root);
=head3 EXAMPLE
my $removed = App::FileSync::Scan::clean_stale_temps(root => '/srv/filesync/master/docs', max_age => 600);
=head3 API SPECIFICATION
=head4 INPUT
{
root => { type => 'string', optional => 1 },
max_age => { type => 'integer', min => 0, optional => 1 },
}
=head4 OUTPUT
{ type => 'integer', min => 0 }
=head3 MESSAGES
None; failures to remove are not errors.
=head3 FORMAL SPECIFICATION
ÎFS
Stale == { p : dom FS | p under root â§ prefix(TEMP, basename p) â§ now â mtime p ⥠age }
dom FS' = dom FS \ Removable(Stale)
result = #(dom FS \ dom FS')
=cut | |||||
| 235 | ||||||
| 236 | sub clean_stale_temps { | |||||
| 237 | 322 | 2439 | my $args = named(\@_, { | |||
| 238 | root => { type => 'string', optional => 1 }, | |||||
| 239 | max_age => { type => 'integer', min => 0, optional => 1 }, | |||||
| 240 | }, 'Scan::clean_stale_temps'); | |||||
| 241 | ||||||
| 242 | 322 | 773 | my $root = $args->{root}; | |||
| 243 | 322 | 2937 | return 0 unless defined $root && length $root && -d $root; | |||
| 244 | ||||||
| 245 | 322 | 1044 | my $max_age = defined $args->{max_age} ? $args->{max_age} : $STALE_TEMP_SECONDS; | |||
| 246 | 322 | 689 | my $now = time; | |||
| 247 | 322 | 387 | my $removed = 0; | |||
| 248 | ||||||
| 249 | # Breadth first with an explicit queue: no recursion depth limit, and | |||||
| 250 | # symlinked directories are never followed because lstat says what they are. | |||||
| 251 | 322 | 609 | my @queue = ($root); | |||
| 252 | 322 | 553 | while (@queue) { | |||
| 253 | 428 | 727 | my $dir = shift @queue; | |||
| 254 | 428 428 | 485 1165 | for my $name (@{ list_dir($dir) || [] }) { | |||
| 255 | 1382 | 7801 | my $path = File::Spec->catfile($dir, $name); | |||
| 256 | 1382 | 6819 | my @stat = Time::HiRes::lstat($path) or next; | |||
| 257 | 1382 | 2555 | if (-d _) { | |||
| 258 | 106 | 210 | push @queue, $path; | |||
| 259 | } elsif ($name =~ $TEMP_NAME && $now - $stat[9] >= $max_age) { | |||||
| 260 | 315 315 315 | 119606 312 677 | no autodie qw(unlink); | |||
| 261 | 0 | 0 | $removed++ if unlink $path; | |||
| 262 | } | |||||
| 263 | } | |||||
| 264 | } | |||||
| 265 | ||||||
| 266 | 322 | 1769 | return $removed; | |||
| 267 | } | |||||
| 268 | ||||||
| 269 | # _walk | |||||
| 270 | # Purpose: scan one directory, recursing into subdirectories. | |||||
| 271 | # Entry: $context is the scan state; $abs/$rel name the directory | |||||
| 272 | # ($rel is '' at the root). | |||||
| 273 | # Exit: returns how many entries inside were recorded, so the caller | |||||
| 274 | # knows whether to record this directory as empty. | |||||
| 275 | # Side effects: fills $context->{manifest}, {skipped} and {stats}. | |||||
| 276 | sub _walk :Private { | |||||
| 277 | my ($context, $abs, $rel) = @_; | |||||
| 278 | ||||||
| 279 | my $names = list_dir($abs); | |||||
| 280 | unless ($names) { | |||||
| 281 | _skip($context, length $rel ? $rel : '.', 'scan.cannot_list', os_error()); | |||||
| 282 | return 0; | |||||
| 283 | } | |||||
| 284 | ||||||
| 285 | my $entries = 0; | |||||
| 286 | for my $name (@{$names}) { | |||||
| 287 | my $child_rel = length $rel ? "$rel/$name" : $name; | |||||
| 288 | next if $context->{ignore} && $context->{ignore}->match($child_rel); | |||||
| 289 | ||||||
| 290 | my $child_abs = File::Spec->catfile($abs, $name); | |||||
| 291 | my @stat = Time::HiRes::lstat($child_abs); | |||||
| 292 | if (!@stat) { | |||||
| 293 | _skip($context, $child_rel, 'scan.cannot_stat', os_error()); | |||||
| 294 | } elsif (-d _) { | |||||
| 295 | # An empty directory is worth syncing; a non-empty one is implied | |||||
| 296 | # by the files inside it. | |||||
| 297 | $entries++; | |||||
| 298 | if (_walk($context, $child_abs, $child_rel) == 0) { | |||||
| 299 | $context->{manifest}{$child_rel} = { t => $TYPE_DIR }; | |||||
| 300 | $context->{stats}{dirs}++; | |||||
| 301 | } | |||||
| 302 | } elsif (-f _) { | |||||
| 303 | my $entry = $file_entry->($context, $child_abs, $child_rel, \@stat) or next; | |||||
| 304 | $context->{manifest}{$child_rel} = $entry; | |||||
| 305 | $context->{stats}{files}++; | |||||
| 306 | $context->{stats}{bytes} += $entry->{s}; | |||||
| 307 | $entries++; | |||||
| 308 | } else { | |||||
| 309 | _skip($context, $child_rel, -l _ ? 'scan.symlink' : 'scan.special'); | |||||
| 310 | } | |||||
| 311 | } | |||||
| 312 | ||||||
| 313 | return $entries; | |||||
| 314 | 315 315 315 | 99307 281 5202 | } | |||
| 315 | ||||||
| 316 | # _skip | |||||
| 317 | # Purpose: record a path the manifest leaves out, and why. | |||||
| 318 | # Entry: $key names the reason in the catalogue; $error is the OS | |||||
| 319 | # reason where there is one. | |||||
| 320 | # Exit: returns 1. | |||||
| 321 | # Side effects: updates $context->{skipped} and the skipped count. | |||||
| 322 | sub _skip :Private { | |||||
| 323 | my ($context, $rel, $key, $error) = @_; | |||||
| 324 | ||||||
| 325 | $context->{skipped}{$rel} = message($key, { error => $error }); | |||||
| 326 | $context->{stats}{skipped}++; | |||||
| 327 | return 1; | |||||
| 328 | 315 315 315 | 86931 237 2522 | } | |||
| 329 | ||||||
| 330 | 1; | |||||
| 331 | ||||||