File Coverage

File:blib/lib/App/Access2CSV/Exporter.pm
Coverage:98.9%

linestmtbrancondsubtimecode
1package App::Access2CSV::Exporter;
2
3
36
36
36
76133
14
170
use strict;
4
36
36
36
21
17
394
use warnings;
5
36
36
36
213
6215
42
use autodie qw(:all);
6
7
36
36
36
46742
14
511
use Config;
8
9# Sub::Private must be in enforce mode before it is loaded, so that
10# private methods still work through $self->method dispatch
11
36
178
BEGIN { $Sub::Private::config{mode} = 'enforce' }
12
13# Inherit i18n() and the protected _croak_i18n/_carp_i18n helpers
14
36
36
36
20
16
43
use parent 'App::Access2CSV::I18N';
15
16
36
36
36
494
14
411
use Encode qw(FB_CROAK find_encoding);
17
36
36
36
29
13
324
use File::Path qw(make_path);
18
36
36
36
22
17
193
use File::Spec;
19
36
36
36
4698
72544
492
use File::Temp;
20
36
35
35
3852
9257
443
use File::Which qw(which);
21
35
34
34
4132
24619
469
use IPC::Run3 qw(run3);
22
34
34
34
45
12
242
use Params::Get qw(get_params);
23
34
34
34
23
18
195
use Params::Validate::Strict qw(validate_strict);
24
34
34
34
25
11
213
use Readonly;
25
34
34
34
26
11
189
use Return::Set qw(set_return);
26
34
34
34
16
19
195
use Scalar::Util qw(blessed);
27
34
34
34
13
20
60
use Sub::Private;
28
34
34
34
1724
15
35
use Sub::Protected;
29
30our $VERSION = '0.001.1';
31
32# Stop Carp from reporting errors against the access-control wrappers
33our @CARP_NOT = qw(Sub::Private Sub::Protected App::Access2CSV::I18N);
34
35# Exit statuses returned by run()
36Readonly::Scalar my $EXIT_OK      => 0;
37Readonly::Scalar my $EXIT_FAILURE => 1;
38
39# The mdbtools programs; mdb-count is only needed for --show-counts
40Readonly::Scalar my $MDB_TABLES   => 'mdb-tables';
41Readonly::Scalar my $MDB_EXPORT   => 'mdb-export';
42Readonly::Scalar my $MDB_COUNT    => 'mdb-count';
43Readonly::Array  my @REQUIRED_PROGRAMS => ($MDB_TABLES, $MDB_EXPORT);
44
45# Output encodings accepted by --encoding
46Readonly::Scalar my $ENC_UTF8     => 'utf8';
47Readonly::Scalar my $ENC_UTF8_BOM => 'utf8-bom';
48Readonly::Scalar my $ENC_CP1252   => 'cp1252';
49Readonly::Array  my @ENCODINGS    => ($ENC_UTF8, $ENC_UTF8_BOM, $ENC_CP1252);
50
51# Taint mode: a value is untainted only after it has been validated, by
52# capturing it with this pattern (anything non-empty without a NUL byte:
53# a NUL cannot be passed to the operating system at all)
54Readonly::Scalar my $UNTAINT_RE => qr/\A([^\x00]+)\z/s;
55
56# Environment variables that can change how a program is started (see
57# perlsec); removed for the mdbtools processes
58Readonly::Array my @UNSAFE_ENV => qw(IFS CDPATH ENV BASH_ENV);
59
60# Encoding objects, looked up once: calling Encode::decode/encode by name
61# repeats the lookup for every line, which made conversion about 4 times
62# slower on large tables.  (Plain lexicals, not Readonly: Readonly's deep
63# copy could interfere with the objects' internals.)
64my $UTF8_CODEC   = find_encoding('UTF-8');
65my $CP1252_CODEC = find_encoding('cp1252');
66
67# Byte order mark written at the start of utf8-bom files (for Excel)
68Readonly::Scalar my $UTF8_BOM     => "\xEF\xBB\xBF";
69
70# Tables Access creates for itself: MSys*, USys* and ~temporary objects
71Readonly::Scalar my $SYSTEM_TABLE_RE => qr/\A(?:MSys|USys|~)/i;
72
73# Characters that are illegal in a file name on at least one common OS
74Readonly::Scalar my $UNSAFE_CHARS_RE => qr/[<>:"\/\\|?*\x00-\x1F\x7F]/;
75
76# Invisible text-direction controls (LRM, RLM, LRE, RLE, PDF, LRO, RLO,
77# LRI, RLI, FSI, PDI).  They can make a file name display as something
78# else ("report<RLO>vsc.exe.csv"), so they are unsafe like other control
79# characters.  Matched both as Perl characters and as UTF-8 bytes, since
80# table names from mdbtools arrive as bytes.
81Readonly::Scalar my $BIDI_CONTROLS_RE => qr/
82          [\x{200E}\x{200F}\x{202A}-\x{202E}\x{2066}-\x{2069}]   # as characters
83        | \xE2 \x80 [\x8E\x8F\xAA-\xAE]                       # as UTF-8: marks, embeddings, overrides
84        | \xE2 \x81 [\xA6-\xA9]                              # as UTF-8: isolates
85/x;
86
87# Device names Windows reserves whatever the extension (CON.csv is illegal)
88Readonly::Scalar my $RESERVED_NAME_RE => qr/\A(?:CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])\z/i;
89
90# Name used when sanitising leaves nothing, and the extension we write
91Readonly::Scalar my $UNNAMED      => 'unnamed';
92Readonly::Scalar my $CSV_SUFFIX   => '.csv';
93
94# Longest file name (before any _2 suffix and ".csv").  Access allows 64
95# characters, so a longer name comes from a damaged or hostile database.
96# 64 characters can still be 256 bytes of UTF-8 (emoji), and most file
97# systems allow 255 bytes per name, so there is a byte limit too, leaving
98# room for "_" plus a collision number and ".csv"
99Readonly::Scalar my $MAX_NAME_CHARS => 64;
100Readonly::Scalar my $MAX_NAME_BYTES => 240;
101
102# Ends option parsing in mdbtools, so a table or database name that
103# starts with "-" is never taken for an option (glib parses options
104# anywhere on the command line, not only before the file name)
105Readonly::Scalar my $END_OF_OPTIONS => '--';
106
107# Temporary files are hidden and live next to the target for atomic rename
108Readonly::Scalar my $TEMP_TEMPLATE => '.access2csv-XXXXXX';
109
110# How much of an unreadable mdb-count answer to quote in the warning
111Readonly::Scalar my $COUNT_SHOWN => 40;
112
113# Shown in the dry run's ROWS column when a table could not be counted
114Readonly::Scalar my $UNKNOWN_COUNT => '?';
115
116# Dry-run table layout
117Readonly::Scalar my $TABLE_COLUMN_WIDTH => 40;
118Readonly::Scalar my $ROWS_COLUMN_WIDTH  => 10;
119Readonly::Scalar my $RULE_WIDTH         => 70;
120
121# Return schema of run(), shared by its two exits
122Readonly::Hash my %RUN_STATUS_SCHEMA => (type => 'integer', min => $EXIT_OK, max => $EXIT_FAILURE);
123
124# Mode bits for new files before the umask is applied
125Readonly::Scalar my $FILE_MODE    => oct('666');
126
127# Default settings; the flat scalar layout is compatible with Object::Configure
128Readonly::Hash my %DEFAULTS => (
129        output_dir  => File::Spec->curdir(),
130        overwrite   => 0,
131        verbose     => 0,
132        dry_run     => 0,
133        show_counts => 0,
134        progress    => 1,
135        encoding    => $ENC_UTF8,
136);
137
138# Constructor argument schema, shared by new() and the POD
139Readonly::Hash my %NEW_SCHEMA => (
140        output_dir  => { type => 'string', min => 1, optional => 1 },
141        tables      => { type => 'arrayref', element_type => 'string', optional => 1 },
142        overwrite   => { type => 'boolean', optional => 1 },
143        verbose     => { type => 'boolean', optional => 1 },
144        dry_run     => { type => 'boolean', optional => 1 },
145        show_counts => { type => 'boolean', optional => 1 },
146        progress    => { type => 'boolean', optional => 1 },
147        encoding    => { type => 'string', memberof => [@ENCODINGS], optional => 1 },
148        logger      => { type => 'object', can => ['debug', 'info', 'warn'], optional => 1 },
149        language    => { type => 'string', optional => 1 },
150);
151
152=encoding utf8
153
154 - 440
=head1 NAME

App::Access2CSV::Exporter - Export the tables of a Microsoft Access database to CSV files

=head1 VERSION

Version 0.001.1

=head1 SYNOPSIS

        use App::Access2CSV::Exporter;

        # 1. The simplest case: every table, into the current directory
        my $exporter = App::Access2CSV::Exporter->new();
        my $status = $exporter->run('shop.accdb');   # 0 = all OK, 1 = some failed

        # 2. Some tables, into a directory, for Excel, replacing old files
        my $exporter = App::Access2CSV::Exporter->new(
                output_dir => 'exports',
                tables     => ['Customers', 'Orders'],
                encoding   => 'utf8-bom',
                overwrite  => 1,
        );
        $exporter->run('shop.accdb');

        # 3. Only look: print the table list and row counts, write nothing
        App::Access2CSV::Exporter->new(dry_run => 1, show_counts => 1)->run('shop.accdb');

        # 4. Inside a larger program: no progress lines, a log, and full
        #    error handling
        use Log::Abstraction;

        my $exporter = App::Access2CSV::Exporter->new({
                output_dir => '/srv/exports',
                progress   => 0,
                logger     => Log::Abstraction->new(logger => '/var/log/export.log'),
        });
        my $status = eval { $exporter->run('/data/shop.accdb') };
        if(!defined $status) {
                die "Nothing was exported: $@";       # for example, the file is missing
        } elsif($status == 1) {
                warn "Some tables were not exported; see the log\n";
        }

=head1 DESCRIPTION

This module does the real work of the C<access2csv> program.  It writes
one CSV file for each table of a Microsoft Access database.

It runs three programs from the B<mdbtools> package: C<mdb-tables> (to
list the tables), C<mdb-export> (to get each table as CSV) and, only when
row counts are wanted, C<mdb-count>.  They must be in your C<PATH>.

Access's own internal tables (names starting with C<MSys>, C<USys> or
C<~>) are skipped.

Each file is first written to a hidden temporary file in the output
directory, and renamed to its real name only when it is complete.  So a
failed export never leaves a half-written CSV file, and an old file is
only replaced by a complete new one.  New files get the usual
permissions (0666 minus your umask).

An exporter can be used for more than one C<run>.  Each C<run> starts
again with the same file names, so running twice gives the same files.

The mdbtools programs are looked up only in absolute C<PATH> directory, so
a program planted in the current directory is never run, and they are
started with a cleaned environment (see L<App::Access2CSV/SECURITY>).
The module works under taint mode (C<perl -T>).  Table names are
printed and logged with control characters escaped, so a hostile name
cannot send escape sequences to your terminal.

Table names and the database path are handed to mdbtools as separate
arguments, never through a shell, and after a C<--> marker.  So names
containing shell characters (C<; | E<gt> $( )>), spaces or newlines, or
starting with C<->, are always treated as names, never as commands or
options.  A table name can never place a file outside the output
directory: C</> and C<\> are replaced, and names cannot start with a dot.

An existing entry at the target name - including a symbolic link, even a
broken one - counts as "already exists".  With C<overwrite>, the link
itself is replaced; the file it pointed to is never written.

The rules for file names are described in
L<App::Access2CSV/How the CSV files are named>.

=head1 ENCODING

=over 4

=item * B<CSV data.>  mdbtools gives UTF-8.  With C<encoding> set to
C<utf8> or C<utf8-bom> the bytes are copied exactly, so every character,
including emoji and non-Latin scripts, is kept.  C<utf8-bom> also writes
the three-byte UTF-8 "byte order mark" first, which Microsoft Excel
needs.  With C<cp1252>, each line is converted to Windows-1252; if a line
has a character that Windows-1252 does not have (for example Greek,
Chinese or an emoji), that table fails and nothing is written for it.

=item * B<Database path and output_dir.>  These are passed to the operating
system unchanged.  Give them as byte strings (the form you get from
C<@ARGV> or C<readdir>).  Non-ASCII names work on systems whose file names
are UTF-8, such as Linux and macOS.

=item * B<Table names> (in C<tables>).  They are compared with the names
that C<mdb-tables> prints, which are UTF-8 bytes.  So give UTF-8 byte
strings, not decoded Perl character strings.  If you have a decoded
string, use C<Encode::encode('UTF-8', $name)> first.  The CSV file name is
made from the same bytes, so non-ASCII names and emoji are kept.

=item * B<Messages.>  All messages are plain ASCII English.

=back

=head1 COMMON PITFALLS

=over 4

=item * B<undef means "use the default", not "false".>  In C<new>,
C<< overwrite => undef >> is the same as not giving C<overwrite> at all.
To switch something off, give C<0>.

=item * B<An empty table list exports nothing.>  C<< tables => undef >>
(or no C<tables>) means "all tables".  C<< tables => [] >> means "no
tables": nothing is exported, and C<run> returns 0.

=item * B<Table names are case-sensitive.>  C<'orders'> does not match the
table C<Orders>.  Names that do not match any table give a warning.

=item * B<A failing logger does not stop the export.>  If the logger dies
(for example, its disk is full), C<run> warns once with "Cannot write to
the log", stops logging for this exporter, and carries on exporting.

=item * B<run can croak.>  C<run> returns 1 when some tables fail, but it
croaks (throws an exception) when nothing can be exported at all: the
database is missing or unreadable, mdbtools is not installed, or the
output directory cannot be created.  Wrap C<run> in C<eval> if your program
must keep going.

=item * B<run may change the show_counts setting.>  If C<show_counts> is
on but C<mdb-count> cannot be found, C<run> warns and switches
C<show_counts> off for this exporter.

=item * B<Settings are copied, not shared.>  C<new> makes its own copy of
the C<tables> list; changing your array later has no effect.  Settings are
not merged in depth: a new C<tables> list replaces the default completely.

=item * B<Warnings go through carp.>  Failed tables and unknown table names
are reported with C<carp>, so they appear on standard error (or in your
C<$SIG{__WARN__}> handler) even when a logger is given.

=item * B<Load the module with use, not require.>  Protection of the
private methods is set up at compile time.  After a run-time
C<require> Perl prints "Too late to run CHECK block" and the protection is
missing.

=back

=head1 METHODS

=head2 new

=head3 Purpose

Make a new exporter with your settings.  Nothing is checked on disk yet.

=head3 Arguments

Named arguments, either as a list or as one hash reference.  All of them
are optional.  An argument whose value is C<undef> is ignored, so its
default is used.

=over 4

=item C<output_dir> - the directory for the CSV files.  Default: the current directory.

=item C<tables> - an array reference of table names to export.  Default:
all tables.  An empty array means no tables.

=item C<overwrite> - true to replace CSV files that already exist.  Default: false.

=item C<verbose> - true to log extra detail.  Default: false.

=item C<dry_run> - true to only print what would be written.  Default: false.

=item C<show_counts> - true to report row counts (needs C<mdb-count>).  Default: false.

=item C<progress> - true to print C<[n/total] table> lines to standard
error.  Default: true.

=item C<encoding> - C<utf8>, C<utf8-bom> or C<cp1252>.  Default: C<utf8>.

=item C<logger> - an object with C<debug>, C<info> and C<warn> methods,
such as a L<Log::Abstraction> object.  Default: no logging.

=item C<language> - a language code such as C<en> for messages.  Default:
taken from the locale (see L<App::Access2CSV::I18N>).

=back

=head3 Returns

A new C<App::Access2CSV::Exporter> object.

=head3 Side Effects

None.  Your C<$@>, C<$!> and C<$_> are left as they were.

=head3 Usage

        my $exporter = App::Access2CSV::Exporter->new({ dry_run => 1 });

=head3 EXAMPLE

        # Export two tables as Windows-1252, replacing older files, with a log
        my $exporter = App::Access2CSV::Exporter->new(
                output_dir => 'out',
                tables     => ['Customers', 'Orders'],
                encoding   => 'cp1252',
                overwrite  => 1,
                logger     => Log::Abstraction->new(logger => 'export.log'),
        );

=head3 API SPECIFICATION

=head4 Input

        {
                output_dir  => { type => 'string', min => 1, optional => 1 },
                tables      => { type => 'arrayref', element_type => 'string', optional => 1 },
                overwrite   => { type => 'boolean', optional => 1 },
                verbose     => { type => 'boolean', optional => 1 },
                dry_run     => { type => 'boolean', optional => 1 },
                show_counts => { type => 'boolean', optional => 1 },
                progress    => { type => 'boolean', optional => 1 },
                encoding    => { type => 'string', memberof => ['utf8', 'utf8-bom', 'cp1252'], optional => 1 },
                logger      => { type => 'object', can => ['debug', 'info', 'warn'], optional => 1 },
                language    => { type => 'string', optional => 1 },
        }

Valid and invalid values (tested in F<t/domain.t>):

        output_dir  valid:   any path of 1 character or more ("0" is valid),
                             including non-ASCII names given as UTF-8 bytes
                    invalid: "" (below the minimum), references
        tables      valid:   undef (all tables), [] (no tables), one name,
                             many names; names are matched byte for byte
                    invalid: anything but an array reference; elements
                             that are references
        booleans    valid:   exactly 1 true TRUE yes on  /  0 false FALSE no off
        (overwrite, invalid: everything else, e.g. "", 2, -1, "True", "Yes",
         verbose,            "0.0", " 1"
         dry_run, show_counts, progress)
        encoding    valid:   exactly utf8, utf8-bom, cp1252
                    invalid: other spellings (UTF8, utf-8, CP1252, " utf8")
        logger      valid:   an object with debug, info and warn methods
                    invalid: an object missing any of them, a plain hash,
                             a string
        language    valid:   any string; "" means "use the environment"
                    invalid: references

=head4 Output

        {
                type => 'object',
                isa  => 'App::Access2CSV::Exporter',
        }

=head3 MESSAGES

All are fatal, and read "Invalid setting: REASON".  The REASON part
comes from L<Params::Validate::Strict> and is not translated; for example:

        +--------------------------------------+------------------------------+-----------------------------+
        | Message                              | Meaning                      | What to do                  |
        +--------------------------------------+------------------------------+-----------------------------+
        | Invalid setting: Unknown parameter   | X is not a known setting     | Remove X, or fix its        |
        |  'X'                                 |                              | spelling                    |
        | Invalid setting: Parameter           | This encoding is not         | Use utf8, utf8-bom or       |
        |  'encoding' (X) must be one of utf8, | supported                    | cp1252                      |
        |  utf8-bom, cp1252                    |                              |                             |
        | Invalid setting: Parameter 'logger'  | logger is not an object      | Give a logger object        |
        |  must be an object                   |                              |                             |
        | Invalid setting: Parameter 'tables'  | tables is not an array       | Give an array reference     |
        |  must be ...                         | reference                    |                             |
        +--------------------------------------+------------------------------+-----------------------------+

=cut
441
442sub new {
443
1809
4398126
        my $class = shift;
444
445        # Validation uses eval internally; the caller's $@ must survive
446
1809
794
        local $@;
447
448        # Drop undefined values so that "not given" means "use the default"
449
1809
2104
        my $args = get_params(undef, \@_) || {};
450
1808
2860
3117
1808
13152
1736
1490
1075
        my %given = map { $_ => $args->{$_} } grep { defined $args->{$_} } keys %{$args};
451
452        # A bad setting is reported in plain words ("Invalid setting: ..."),
453        # not with Params::Validate::Strict's internal prefix and location
454
1808
1808
910
1573
        my $params = eval { validate_strict(schema => { %NEW_SCHEMA }, input => \%given) }
455                or $class->_croak_i18n('invalid_setting', { params => [_validation_reason($@)] });
456
457        # Copy the table list so later changes by the caller cannot affect us
458
1716
46
473168
104
        $params->{tables} = [ @{ $params->{tables} } ] if $params->{tables};
459
460        # The output directory is the invoking user's own choice; it is only used
461        # as a directory name, so it is untainted here (see _untaint)
462
1716
1389
        $params->{output_dir} = _untaint($params->{output_dir}) if defined $params->{output_dir};
463
464
1716
1716
1239
10857
        my $self = bless { %DEFAULTS, %{$params}, used_names => {}, next_suffix => {}, programs => {} }, $class;
465
1716
14000
        return set_return($self, { type => 'object' });
466}
467
468 - 682
=head2 run

=head3 Purpose

Export the selected tables of one database to CSV files.  In dry-run mode,
only print what would be exported.

=head3 Arguments

=over 4

=item C<database> (string, required) - the path of the C<.mdb> or C<.accdb> file.
The name is taken literally: C<-> is a file called C<->.  (Reading from
standard input is a feature of the command-line program; see
L<App::Access2CSV/Reading the database from standard input>.)

=back

You can give it on its own, C<< $exporter->run('shop.accdb') >>, or as a
hash reference, C<< $exporter->run({ database => 'shop.accdb' }) >>.

=head3 Returns

C<0> if every selected table was exported, or in dry-run mode.
C<1> if at least one table was not exported (the others were).

=head3 Side Effects

=over 4

=item * Creates the output directory if needed (not in dry-run mode, and
not when no table is selected).

=item * Writes one CSV file per table (not in dry-run mode).

=item * Prints progress lines to standard error, if C<progress> is on.

=item * Prints the dry-run list to standard output, in dry-run mode.

=item * Sends messages to the logger, if there is one.

=item * Warns (with C<carp>) about each table that failed, about unknown
names in C<tables>, and about a missing C<mdb-count>.

=item * Croaks, before writing anything, if the database cannot be read, a
needed mdbtools program is missing, C<mdb-tables> fails, or the output
directory cannot be created.

=item * Switches C<show_counts> off for this exporter if C<mdb-count> is
missing.

=item * While it runs, handles the signals INT, QUIT, TERM and HUP (only
those you have not set a handler for yourself; they are restored when
C<run> returns).  Any of them stops the run: the table being exported is
discarded - its temporary file deleted, any old CSV file left as it was -
no further table is started, and C<run> croaks.  Pressing Ctrl-C, which
also stops the mdbtools program, has the same effect.

=item * Leaves your C<$@>, C<$!>, C<$?>, C<$_>, C<$.> and any pending C<alarm>
as they were (except that a croak sets C<$@> in your C<eval>, as usual).

=back

=head3 Usage

        exit $exporter->run('shop.accdb');

=head3 EXAMPLE

        my $exporter = App::Access2CSV::Exporter->new(output_dir => 'out');

        # eval catches the fatal errors; the return value covers the rest
        my $status = eval { $exporter->run('shop.accdb') };
        if(!defined $status) {
                print STDERR "Nothing was exported: $@";
        } elsif($status) {
                print STDERR "Some tables failed; see the warnings above\n";
        } else {
                print "Done\n";
        }

=head3 API SPECIFICATION

=head4 Input

        {
                database => {
                        type     => 'string',
                        min      => 1,
                        optional => 0,
                },
        }

Valid and invalid values (tested in F<t/domain.t>):

        database    valid:   a readable regular file
                    invalid: "" (below the 1-character minimum), undef,
                             a missing file, a directory, a device or FIFO,
                             an unreadable file
                    edges:   each part of the path may be up to 255 bytes;
                             256 gives "File name too long"

The table names that mdbtools reports are data, not arguments, but they
have limits of their own:

        length      a CSV file name is the table name plus ".csv".  Access
                    allows at most 64 characters, so longer names (only from
                    a damaged or hostile database) are shortened to 64
                    characters, and also to at most 240 bytes of UTF-8 (64
                    emoji would be 256 bytes; most file systems allow 255).
                    A name is never cut inside a character, nor between a
                    letter and its accents or inside a joined emoji.
                    Shortened names that clash are numbered as usual.
        characters  non-ASCII letters, emoji, joined emoji, combining marks
                    and right-to-left text are kept byte for byte.
                    Characters that are unsafe in file names - including
                    invisible text-direction controls such as U+202E - are
                    replaced by "_".  So is each byte that is not part of
                    valid UTF-8 (only a damaged database has them; macOS
                    refuses such file names).
        collisions  the first name has no suffix, then _2, _3, ... _10 ...
        cp1252      U+00FF and the Euro sign convert; U+0100 and above
                    (except the few Windows-1252 symbols), the C1 controls
                    U+0080-U+009F and emoji make the table fail.

=head4 Output

        {
                type => 'integer',
                min  => 0,
                max  => 1,
        }

=head3 MESSAGES

"fatal" means C<run> croaks and nothing is exported.  "per table" means
only that table fails; C<run> warns, logs, and carries on.

        +-----------------------------------------+------------------------------+-------------------------------+
        | Message                                 | Meaning                      | What to do                    |
        +-----------------------------------------+------------------------------+-------------------------------+
        | Interrupted by SIGx: stopped, and the   | Ctrl-C, Ctrl-\\, kill or a   | Run again; tables finished    |
        |  table being exported was discarded     | closed terminal stopped the  | before the interruption are   |
        |  (fatal)                                | run                          | complete                      |
        | run() must be called on an object       | run was called on the class  | Call new() first, then run()  |
        |  created by new() (fatal)               | or on something that is not  | on the object it returns      |
        |                                         | an exporter                  |                               |
        | Cannot read database F: E (fatal)       | F does not exist, or cannot  | Check the path                |
        |                                         | be reached; E is the reason  |                               |
        |                                         | from the operating system    |                               |
        | Database F is not a regular file (fatal)| F is a directory or a device    | Give the database file        |
        | Database F is not readable (fatal)      | No permission to read F      | Fix the permissions           |
        |                                         | (never happens for root)     |                               |
        | Required program not found in PATH: P   | mdbtools is not installed,   | Install mdbtools, or fix PATH |
        |  (fatal)                                | or not in PATH               |                               |
        | mdb-tables failed with exit status N: E | mdbtools cannot read the     | Check that F is a real Access |
        |  (fatal)                                | file                         | database                      |
        | Cannot create output directory D: E     | The directory cannot be made;   | Check permissions and path    |
        |  (fatal)                                | E is the reason for D itself |                               |
        |                                         | (e.g. "Not a directory" when |                               |
        |                                         | a file is in the way)        |                               |
        | Cannot count the rows of T: E (warning) | mdb-count failed for table T | The table is still exported   |
        |                                         | (only with show_counts)      | (dry run: count shown as "?") |
        |  ... mdb-count printed no number: "X"   | mdb-count's answer was not   | As above; X shows what it     |
        |                                         | just a number                | printed                       |
        | Tables not found in database: T         | Names in tables are not in   | Check spelling and case       |
        |  (warning)                              | the database                 |                               |
        | mdb-count not found in PATH; row counts | show_counts is on, but       | Install mdb-count, or turn    |
        |  are unavailable (warning)              | mdb-count is missing         | show_counts off               |
        | FAILED: T: E (warning, logged)          | Table T was not exported,    | See E, one of the messages    |
        |                                         | because of E                 | below                         |
        | Output file already exists: F (use      | F exists (a symbolic link,   | Set overwrite, or use another |
        |  --overwrite to replace it) (per table) | even a broken one, counts)   | output_dir                    |
        |                                         | and overwrite is off         |                               |
        | mdb-export failed with exit status N: E | mdbtools could not read this | Check the table in Access     |
        |  (per table)                            | table                        |                               |
        | P was killed by signal N (per table,    | The program was stopped from | Check memory and system       |
        |  or fatal for mdb-tables)               | outside                      | limits                        |
        | P could not be run: E (per table, or    | The program was found but    | Check its permissions and     |
        |  fatal for mdb-tables)                  | could not be started         | that it is a real program     |
        | Table T, line N: cannot be represented  | A character is not in        | Use utf8 or utf8-bom          |
        |  in cp1252 (per table)                  | Windows-1252                 |                               |
        | Table T, line N: output of mdb-export   | mdbtools gave bytes that are | Check the MDB_ICONV setting   |
        |  is not valid UTF-8 (per table)         | not UTF-8                    |                               |
        | Cannot write F: E (per table)           | The file could not be        | Check permissions and free    |
        |                                         | written or renamed into place| disk space                    |
        | Cannot write to the log: E (warning,    | The logger failed.  Exports  | Check the log's disk or       |
        |  once)                                  | go on; logging stops         | destination                   |
        +-----------------------------------------+------------------------------+-------------------------------+

=head3 PSEUDOCODE

        check the argument
        stop (croak) unless the database is a readable file
        find mdb-tables and mdb-export (croak if missing),
             and mdb-count if row counts are wanted (warn if missing)
        forget the file names given out by any earlier run
        tables := the sorted user tables, filtered by "tables"
                  (warn about names that are not found)
        if dry run:
                print the table -> file list (row count "?" with a warning
                      if a count fails)
                return 0
        if there are tables to export:
                create the output directory (croak if that fails)
        for each table:
                print "[n/total] table" if progress is on
                try to export the table
                if that failed: warn, log, and count the failure
                (a failed row count after the file is in place is only a
                 warning; the table still counts as exported)
        log the summary
        return 1 if any table failed, else 0

=cut
683
684sub run {
685
401
68098
        my $self = shift;
686
687        # State machine guard: run() is only a transition out of READY, which
688        # only new() can create.  Refuse anything else before doing any work.
689        # (Reported through the class: $self may not even be an object.)
690
401
1566
        blessed($self) && $self->isa(__PACKAGE__) or __PACKAGE__->_croak_i18n('needs_object');
691
692        # File tests, evals and child processes below would otherwise leave
693        # their marks in the caller's $@ and $!
694
397
1771
        local ($@, $!);
695
696        # An undef database is a missing one, not a file called "".  Work on a
697        # copy: get_params hands back the caller's own hash when given one.
698        # (Params::Get either dies or returns a hash reference - proved in
699        # t/path.t - so no test of what it returned is needed.)
700
397
397
248
431
        my $input = { %{ get_params('database', \@_) } };
701
391
4077
        delete $input->{database} unless defined $input->{database};
702
391
642
        my $params = validate_strict(
703                schema => { database => { type => 'string', min => 1 } },
704                input  => $input,
705        );
706
382
14100
        my $database = $params->{database};
707
708        # Fail fast, before any output, on problems that affect every table
709
382
697
        $self->_check_database($database)
710                ->_verify_dependencies()
711                ->_reset_names();
712
713        # Stopping part-way must behave like a failed transaction: the table
714        # being exported is discarded (its temporary file deleted) and no
715        # further table is started.  Perl's default action for these signals
716        # is to exit at once, skipping the clean-up, so while run() is active
717        # they raise an exception instead.  A handler the caller has set is
718        # left alone; everything is restored when run() returns.
719
312
301
        local $self->{interrupted};
720
312
312
206
513
        my @ours = @{ $self->_interrupt_signals() };
721        local @SIG{@ours} = (sub {
722
1
32725
                $self->{interrupted} = $_[0];
723
1
14
                die $self->_printable($self->i18n('interrupted', { params => [$_[0]] })), "\n";
724
312
3863
        }) x @ours;
725
726        # Premise: the database is now known to be a readable regular file, and
727        # it is only ever passed to mdbtools as one list argument after "--".
728        # Conclusion: it is safe to untaint.
729
312
351
        $database = _untaint($database);
730
731
312
538
        my $tables = $self->_select_tables($self->_get_tables($database));
732
733        # Guard clause: a dry run must not touch the file system, so it leaves
734        # before mkdir.  Premise: _dry_run writes nothing that can fail a table.
735        # Conclusion: a dry run always succeeds.
736
296
609
        if($self->{dry_run}) {
737
63
226
                $self->_dry_run($database, $tables);
738
63
289
                return set_return($EXIT_OK, { %RUN_STATUS_SCHEMA });
739        }
740
741        # The output directory is only made when there is something to put in it;
742        # with no tables selected the run goes straight to the summary
743
233
233
230
728
        my $failed = (@{$tables} ? $self->_make_output_dir() : $self)->_export_all($database, $tables);
744
224
1462
        return set_return($failed ? $EXIT_FAILURE : $EXIT_OK, { %RUN_STATUS_SCHEMA });
745}
746
747# _check_database
748# Purpose:        Make sure the database is a readable regular file.
749# Entry Criteria: $database is a defined, non-empty path.
750# Exit Status:    Returns $self for chaining; croaks otherwise.
751# Side Effects:   stat()s the file; sets $!.
752sub _check_database :Private {
753
386
4137
        my ($self, $database) = @_;
754
755        # The stat result is reused via "_" so the file is only examined once;
756        # $! is captured straight away because later calls may overwrite it
757
386
1334
        if(!-e $database) {
758
36
214
                $self->_croak_i18n('database_not_found', { params => [$database, "$!"] });
759        }
760
350
370
        $self->_croak_i18n('database_not_file', { params => [$database] }) unless -f _;
761
337
531
        $self->_croak_i18n('database_unreadable', { params => [$database] }) unless -r _;
762
763
331
646
        return $self;
764
34
34
34
13342
14
310
}
765
766# _verify_dependencies
767# Purpose:        Locate the mdbtools programs in PATH.
768# Entry Criteria: None.
769# Exit Status:    Returns $self; croaks if a required program is missing.
770# Side Effects:   Sets $self->{programs}; may switch off show_counts (with a
771#                 warning) when mdb-count is unavailable; logs at debug level.
772sub _verify_dependencies :Private {
773
354
3303
        my $self = shift;
774
775
354
199
        my %programs;
776
354
823
        foreach my $program (@REQUIRED_PROGRAMS) {
777
684
1710
                $programs{$program} = $self->_find_program($program)
778                        or $self->_croak_i18n('program_missing', { params => [$program] });
779        }
780
781        # mdb-count is only needed for row counts, so its absence is not fatal.
782        # _find_program returns a path or false, so one branch decides both
783        # "store it" and "switch counts off" (nothing is stored and removed).
784
315
668
        if($self->{show_counts}) {
785
47
62
                if(my $path = $self->_find_program($MDB_COUNT)) {
786
35
58
                        $programs{$MDB_COUNT} = $path;
787                } else {
788
12
21
                        $self->{show_counts} = 0;
789
12
40
                        $self->_warn('no_row_counter');
790                }
791        }
792
793
315
325
        $self->{programs} = \%programs;
794
315
586
        return $self;
795
34
34
34
7297
18
126
}
796
797# _find_program
798# Purpose:        Look up one program in PATH and note where it was found.
799# Entry Criteria: $program is a bare program name.
800# Exit Status:    Returns the full path, or undef if not found.
801# Side Effects:   Logs the location at debug level when --verbose is on.
802sub _find_program :Private {
803
730
2729
        my ($self, $program) = @_;
804
805        # Only absolute paths are trusted.  A relative entry in PATH (".", or
806        # an empty one) would run whatever file of that name is in the current
807        # directory - a classic way to plant a program.
808
730
715
2129
37691
        my ($path) = grep { defined && File::Spec->file_name_is_absolute($_) } which($program);
809
810        # An absolute path to an existing program: safe to untaint
811
730
937
        $path = _untaint($path) if defined $path;
812
730
1184
        if($path && $self->{verbose}) {
813
8
13
                $self->_log(debug => 'program_found', { params => [$program, $path] });
814        }
815
730
1044
        return $path;
816
34
34
34
5740
14
123
}
817
818# _reset_names
819# Purpose:        Forget file names allocated by a previous run() so that
820#                 running the same exporter twice gives the same names.
821# Entry Criteria: None.
822# Exit Status:    Returns $self.
823# Side Effects:   Empties $self->{used_names}.
824sub _reset_names :Private {
825
312
1076
        my $self = shift;
826
827
312
271
        $self->{used_names} = {};
828
312
291
        $self->{next_suffix} = {};
829
312
203
        return $self;
830
34
34
34
5099
19
89
}
831
832# _get_tables
833# Purpose:        List the user tables in the database.
834# Entry Criteria: _verify_dependencies() has run.
835# Exit Status:    Returns an arrayref of table names, sorted; croaks if
836#                 mdb-tables fails.
837# Side Effects:   Runs mdb-tables.
838sub _get_tables :Protected {
839
308
1394
        my ($self, $database) = @_;
840
841        # -1 puts one table per line, so names containing spaces survive;
842        # "--" stops a database path starting with "-" being read as an option
843
308
299
        my $stdout = '';
844
308
662
        $self->_run_program($MDB_TABLES, ['-1', $END_OF_OPTIONS, $database], \$stdout);
845
846        # \r? copes with mdbtools builds that emit CRLF line endings; a program
847        # that printed nothing may leave $stdout undefined
848        # Table names are untainted: they are only used as one list argument
849        # after "--", and in file names only after _csv_filename has made them
850        # safe
851
292
1524
401541
62541
1339
194730
        my @tables = sort map { _untaint($_) } grep { length($_) && !$self->_is_system_table($_) } split /\r?\n/, $stdout // '';
852
292
6560
        return \@tables;
853
34
34
34
5556
38
110
}
854
855# _is_system_table
856# Purpose:        Decide whether a table is Access's own rather than the user's.
857# Entry Criteria: $table is a table name.
858# Exit Status:    Returns 1 for system tables, 0 otherwise.
859# Side Effects:   None.  Protected so that subclasses can widen the filter.
860sub _is_system_table :Protected {
861
401550
732056
        my ($self, $table) = @_;
862
863
401550
316337
        return ($table =~ $SYSTEM_TABLE_RE) ? 1 : 0;
864
34
34
34
4671
8
113
}
865
866# _select_tables
867# Purpose:        Apply the --table filter to the list of tables.
868# Entry Criteria: $tables is the arrayref from _get_tables().
869# Exit Status:    Returns an arrayref, in database (sorted) order.
870# Side Effects:   Warns and logs about requested tables that do not exist.
871sub _select_tables :Private {
872
302
2025
        my ($self, $tables) = @_;
873
874
302
793
        return $tables unless $self->{tables};
875
876        # Matching is exact, as mdb-export itself is case-sensitive
877
35
87
35
59
166
54
        my %available = map { $_ => 1 } @{$tables};
878
35
1050
35
48
519
56
        my %wanted    = map { $_ => 1 } @{ $self->{tables} };
879
880
35
51
85
120
        my @missing = sort grep { !$available{$_} } keys %wanted;
881
35
61
        if(@missing) {
882
12
65
                $self->_warn('unknown_tables', { params => [join(', ', @missing)], count => scalar(@missing) });
883        }
884
885
35
87
35
42
102
34
        return [ grep { $wanted{$_} } @{$tables} ];
886
34
34
34
5859
17
99
}
887
888# _make_output_dir
889# Purpose:        Create the output directory (and parents) if necessary.
890# Entry Criteria: Not in dry-run mode.
891# Exit Status:    Returns $self; croaks if the directory cannot be created.
892# Side Effects:   Creates directories.
893sub _make_output_dir :Private {
894
231
1787
        my $self = shift;
895
896
231
384
        my $dir = $self->{output_dir};
897
231
1627
        return $self if -d $dir;
898
899        # Ask File::Path to report errors rather than carp/croak on its own,
900        # so the message can be translated and names the directory we wanted
901
177
19158
        make_path($dir, { error => \my $errors });
902
177
177
263
892
        if(@{$errors} || !-d $dir) {
903                # File::Path may also report a parent (e.g. "File exists" for a
904                # plain file in the way); the reason for $dir itself, or failing
905                # that the last one, is what explains the failure
906
18
24
18
17
79
24
                my ($mine) = grep { exists $_->{$dir} } @{$errors};
907
18
6
2
34
5
3
                my $detail = $mine ? $mine->{$dir} : (@{$errors} ? (values %{ $errors->[-1] })[0] : undef);
908
18
78
                $self->_croak_i18n('mkdir_failed', { params => [$dir, $detail || "$!"] });
909        }
910
159
441
        return $self;
911
34
34
34
5711
16
119
}
912
913# _export_all
914# Purpose:        Export each table, carrying on past individual failures.
915# Entry Criteria: The output directory exists.
916# Exit Status:    Returns the number of tables that failed.
917# Side Effects:   Writes CSV files; prints progress; warns; logs.
918sub _export_all :Private {
919
229
1732
        my ($self, $database, $tables) = @_;
920
921
229
229
160
245
        my $total  = scalar @{$tables};
922
229
208
        my $failed = 0;
923
924        # eval below would otherwise overwrite the caller's $@
925
229
160
        local $@;
926
927        # An index loop, not each(), which shares the array's iterator with the
928        # caller and would silently skip tables if it was already part-way
929
229
229
204
349
        foreach my $index (0 .. $#{$tables}) {
930
1391
993
                my $table = $tables->[$index];
931                # Progress goes to STDERR so that STDOUT can be redirected cleanly
932
1391
1204
                if($self->{progress}) {
933
53
307
                        print STDERR $self->_printable($self->i18n('progress', { params => [$index + 1, $total, $table] })), "\n";
934                }
935
936                # One bad table should not stop the rest from being exported
937
1391
1391
1315
704
1634
18784
                next if eval { $self->_export_table($database, $table); 1 };
938
939                # ... but an interruption stops them all: the failed table has been
940                # discarded (its temporary file went with the exception), and no
941                # further table is started
942
76
18284
                $self->_croak_i18n('interrupted', { params => [$self->{interrupted}] }) if $self->{interrupted};
943
944
74
138
                my $error = $@ || 'Unknown error';
945
74
87
                chomp $error;
946
74
62
                ++$failed;
947
74
225
                $self->_warn('export_failed', { params => [$table, $error] });
948        }
949
950
227
759
        $self->_log(info => 'summary', { params => [$total, $failed], count => $total });
951
227
556
        return $failed;
952
34
34
34
5976
29
91
}
953
954# _export_table
955# Purpose:        Export one table to its CSV file.
956# Entry Criteria: _verify_dependencies() and _make_output_dir() have run.
957# Exit Status:    Returns $self; croaks on any failure, leaving no partial file.
958# Side Effects:   Runs mdb-export (and mdb-count); creates or replaces a file.
959sub _export_table :Protected {
960
1391
5165
        my ($self, $database, $table) = @_;
961
962
1391
1616
        my $outfile = File::Spec->catfile($self->{output_dir}, $self->_csv_filename($table));
963
964        # Check before exporting so that we do not waste time on a big table
965        # -l as well as -e: a dangling symlink is an existing entry too, and
966        # must not be silently replaced.  The overwrite flag is tested first:
967        # when it is set the answer is already known, so no file test is needed.
968
1391
11068
        if(!$self->{overwrite} && (-e $outfile || -l $outfile)) {
969
24
115
                $self->_croak_i18n('output_exists', { params => [$outfile] });
970        }
971
972        # Write into a temporary file next to the target; it is deleted
973        # automatically if anything below croaks
974
1367
2684
        my $tmp = File::Temp->new(DIR => $self->{output_dir}, TEMPLATE => $TEMP_TEMPLATE, UNLINK => 1);
975
1365
129577
        binmode $tmp, ':raw';
976
977
1365
28273
        if($self->{encoding} eq $ENC_CP1252) {
978
45
104
                $self->_export_transcoded($database, $table, $tmp);
979        } else {
980                # The BOM must reach the file before mdb-export starts writing to
981                # the same descriptor, hence the explicit flush
982
1320
19
996
72
                print {$tmp} $UTF8_BOM if $self->{encoding} eq $ENC_UTF8_BOM;
983
1320
1692
                $tmp->flush() or $self->_croak_i18n('write_failed', { params => [$outfile, "$!"] });
984
1317
1596
                $self->_run_program($MDB_EXPORT, [$END_OF_OPTIONS, $database, $table], $tmp);
985        }
986
987
1320
4751
        $self->_install_file($tmp, $outfile);
988
989        # The file is now in place, so the table has been exported.  Row counts
990        # are optional extras: if counting fails it is only a warning (see
991        # _try_count_rows), and the export is logged without a count.
992
1316
1668
        my $rows = $self->{show_counts} ? $self->_try_count_rows($database, $table) : undef;
993
1316
977
        if(defined $rows) {
994
31
271
                $self->_log(info => 'exported_rows', { params => [$table, $outfile, $rows], count => $rows });
995        } else {
996
1285
1955
                $self->_log(info => 'exported', { params => [$table, $outfile] });
997        }
998
1316
1737
        return $self;
999
34
34
34
6435
9
99
}
1000
1001# _export_transcoded
1002# Purpose:        Export a table and convert it from UTF-8 to Windows-1252.
1003# Entry Criteria: $out is an open, raw, writable filehandle.
1004# Exit Status:    Returns $self; croaks on invalid UTF-8 or on a character
1005#                 that has no cp1252 equivalent (rather than silently
1006#                 replacing it with '?').
1007# Side Effects:   Runs mdb-export into a second temporary file.
1008sub _export_transcoded :Private {
1009
51
1251
        my ($self, $database, $table, $out) = @_;
1010
1011        # Reading the spool changes $. and the evals change $@; keep the
1012        # caller's values
1013
51
69
        local $.;
1014
51
42
        local $@;
1015
1016        # Spool to disk rather than memory, so huge tables do not exhaust RAM
1017
51
104
        my $spool = File::Temp->new(DIR => $self->{output_dir}, TEMPLATE => $TEMP_TEMPLATE, UNLINK => 1);
1018
51
3796
        binmode $spool, ':raw';
1019
51
729
        $self->_run_program($MDB_EXPORT, [$END_OF_OPTIONS, $database, $table], $spool);
1020
43
295
        seek $spool, 0, 0;
1021
1022        # Convert line by line; $. gives the user a line number to look at
1023
43
4534
        while(my $line = <$spool>) {
1024
81
81
64
242
                my $chars = eval { $UTF8_CODEC->decode($line, FB_CROAK) };
1025
81
344
                $self->_croak_i18n('invalid_utf8', { params => [$table, $.] }) unless defined $chars;
1026
1027
74
74
52
219
                my $bytes = eval { $CP1252_CODEC->encode($chars, FB_CROAK) };
1028
74
140
                $self->_croak_i18n('unmappable', { params => [$table, $., $ENC_CP1252] }) unless defined $bytes;
1029
1030
63
63
41
228
                print {$out} $bytes;
1031        }
1032
1033        # Close the spool explicitly once it has been read.  (On a croak above,
1034        # File::Temp's destructor closes and deletes it.)
1035
25
83
        close $spool;
1036
25
956
        return $self;
1037
34
34
34
6530
20
91
}
1038
1039# _install_file
1040# Purpose:        Move a finished temporary file to its final name.
1041# Entry Criteria: $tmp is a File::Temp holding the complete CSV.
1042# Exit Status:    Returns $self; croaks if the rename or chmod fails.
1043# Side Effects:   Replaces $outfile; the temporary file is no longer
1044#                 auto-deleted.
1045sub _install_file :Private {
1046
1315
6631
        my ($self, $tmp, $outfile) = @_;
1047
1048        # The eval must not overwrite the caller's $@
1049
1315
924
        local $@;
1050
1051        # File::Temp creates files as 0600; give the CSV the permissions a
1052        # normal open() would have, i.e. 0666 less the umask
1053
1315
766
        my $ok = eval {
1054
1315
1840
                close $tmp;
1055
1315
33143
                chmod $FILE_MODE & ~umask(), $tmp->filename();
1056
1315
40752
                rename $tmp->filename(), $outfile;
1057
1309
41431
                1;
1058        };
1059
1315
3829
        $self->_croak_i18n('write_failed', { params => [$outfile, _os_error($@)] }) unless $ok;
1060
1061
1309
1377
        $tmp->unlink_on_destroy(0);
1062
1309
3511
        return $self;
1063
34
34
34
5186
20
80
}
1064
1065# _count_rows
1066# Purpose:        Ask mdb-count how many rows a table has.
1067# Entry Criteria: $self->{programs}{'mdb-count'} is set.
1068# Exit Status:    Returns a non-negative integer; croaks if mdb-count fails.
1069# Side Effects:   Runs mdb-count.
1070sub _count_rows :Private {
1071
57
549
        my ($self, $database, $table) = @_;
1072
1073
57
81
        my $stdout = '';
1074
57
200
        $self->_run_program($MDB_COUNT, [$END_OF_OPTIONS, $database, $table], \$stdout);
1075
1076        # mdb-count prints just the number (perhaps with spaces around it).
1077        # Anything else - nothing, "-5", an error text - is not a count, and
1078        # must not quietly become 0: it is reported (as a warning, by
1079        # _try_count_rows)
1080
50
522
        my ($rows) = ($stdout // '') =~ /\A\s*(\d+)\s*\z/;
1081
50
130
        defined($rows) or $self->_croak_i18n('count_unreadable', { params => [$self->_printable(substr($stdout // '', 0, $COUNT_SHOWN))] });
1082
42
152
        return $rows;
1083
34
34
34
5465
18
89
}
1084
1085# _try_count_rows
1086# Purpose:        Count a table's rows, where a failure is only a warning.
1087#                 Row counts are optional extras: they must never make an
1088#                 exported table count as failed, nor end a dry run.
1089# Entry Criteria: $self->{programs}{'mdb-count'} is set.
1090# Exit Status:    Returns the count, or undef if it could not be had.
1091# Side Effects:   Runs mdb-count; on failure warns and logs "Cannot count
1092#                 the rows of T: E".  An interruption is not a failure to
1093#                 count: it is passed on, to stop the run.
1094sub _try_count_rows :Private {
1095
61
439
        my ($self, $database, $table) = @_;
1096
1097
61
65
        local $@;
1098
61
61
67
154
        my $rows = eval { $self->_count_rows($database, $table) };
1099
61
2137
        return $rows if defined $rows;
1100
1101
15
21
        die $@ if $self->{interrupted};
1102
14
10
        my $error = $@;
1103
14
16
        chomp $error;
1104
14
27
        $self->_warn('count_failed', { params => [$table, $error] });
1105
14
21
        return;
1106
34
34
34
5844
15
75
}
1107
1108# _run_program
1109# Purpose:        Run an mdbtools program and check that it succeeded.
1110#                 Shared by every call to mdbtools, so that failures are
1111#                 reported consistently.
1112# Entry Criteria: $name is a key of $self->{programs}; $args is an arrayref;
1113#                 $stdout is a scalar ref or a filehandle for run3().
1114# Exit Status:    Returns $self; croaks if the program exits non-zero or
1115#                 dies from a signal.
1116# Side Effects:   Runs a child process; writes to $stdout; sets $?.
1117sub _run_program :Private {
1118
1721
7601
        my ($self, $name, $args, $stdout) = @_;
1119
1120        # run3 sets $?, and reads captured output back from a temporary file,
1121        # which changes the handle $. refers to; keep the caller's values
1122
1721
1744
        local ($?, $.);
1123
1124        # Start the program in a clean environment (as perlsec asks, and as
1125        # taint mode requires): PATH keeps only absolute directory, and variables
1126        # that can change how a program is started are removed
1127        # (File::Spec->path and path_sep, not ":": Windows separates PATH with
1128        # ";" and its paths contain ":", as in C:\\)
1129
1721
31646
33230
12325
12626
27439
        local $ENV{PATH} = join($Config{path_sep}, map { _untaint($_) } grep { length && File::Spec->file_name_is_absolute($_) } File::Spec->path());
1130
1721
3764
        local @ENV{@UNSAFE_ENV};
1131
1721
12461
        delete @ENV{@UNSAFE_ENV};
1132
1133        # A list (not a string) is passed, so no shell ever sees the file or
1134        # table name and quoting cannot be abused
1135
1721
12247
        my $stderr = '';
1136
1721
1721
1281
2260
        run3([$self->{programs}{$name}, @{$args}], \undef, $stdout, \$stderr);
1137
1138        # Distinguish "never started" (-1) and a signal from an ordinary
1139        # non-zero exit status; $! only means something in the first case
1140
1715
54876234
        my ($status, $reason) = ($?, "$!");
1141
1715
1442
        chomp $stderr;
1142
1715
1958
        if($status == -1) {
1143
6
13
                $self->_croak_i18n('program_not_run', { params => [$name, $reason] });
1144        }
1145
1709
1845
        if($status & 127) {
1146                # While system() waits, Perl ignores INT and QUIT in this process,
1147                # so Ctrl-C shows up only as the child dying of SIGINT.  Treat
1148                # that as the user stopping the whole run, not as a bad table.
1149
14
213
                my $signal = (split ' ', $Config{sig_name})[$status & 127] // '';
1150
14
75
                if($signal eq 'INT' || $signal eq 'QUIT') {
1151
3
5
                        $self->{interrupted} = $signal;
1152
3
16
                        $self->_croak_i18n('interrupted', { params => [$signal] });
1153                }
1154
11
31
                $self->_croak_i18n('program_signalled', { params => [$name, $status & 127] });
1155        }
1156
1695
1482
        if($status) {
1157
33
207
                $self->_croak_i18n('program_failed', { params => [$name, $status >> 8, $stderr] });
1158        }
1159
1662
19364
        return $self;
1160
34
34
34
7135
12
92
}
1161
1162# _csv_filename
1163# Purpose:        Turn a table name into a safe, unique CSV file name.
1164# Entry Criteria: $table is a table name (possibly empty).
1165# Exit Status:    Returns a file name (no directory) ending in ".csv".
1166# Side Effects:   Records the name in $self->{used_names}.
1167#
1168# Names are compared case-insensitively, because Windows and macOS file
1169# systems are, so "Orders" and "ORDERS" do not overwrite each other.
1170# The loop (rather than a single suffix) guarantees uniqueness even when
1171# another table is literally called "Orders_2".
1172sub _csv_filename :Protected {
1173
17111
66815
        my ($self, $table) = @_;
1174
1175
17111
7447
        my $name = $table // '';
1176
1177        # macOS only accepts file names that are valid UTF-8 (anything else
1178        # fails with "Illegal byte sequence").  mdbtools gives UTF-8, so other
1179        # bytes come from a damaged database: each one becomes "_", on every
1180        # system, so a table gets the same file name everywhere
1181
17111
8248
        if(!utf8::is_utf8($name)) {
1182
17109
263
15423
173
                $name = $UTF8_CODEC->encode($UTF8_CODEC->decode($name, sub { '_' }));
1183        }
1184
1185        # Replace characters that are illegal somewhere, then strip leading
1186        # and trailing whitespace; Windows also silently drops trailing dots
1187
17111
23741
        $name =~ s/$UNSAFE_CHARS_RE/_/g;
1188
17111
11450
        $name =~ s/$BIDI_CONTROLS_RE/_/g;
1189
1190        # C1 controls (U+0080-U+009F; U+009B acts like ESC [ on terminals).
1191        # In a byte string they must be matched as their UTF-8 form, because a
1192        # bare [\x80-\x9F] would also hit the continuation bytes of ordinary
1193        # characters such as the Euro sign (E2 82 AC)
1194
17111
7845
        if(utf8::is_utf8($name)) {
1195
2
2
                $name =~ s/[\x{80}-\x{9F}]/_/g;
1196        } else {
1197
17109
6680
                $name =~ s/\xC2[\x80-\x9F]/_/g;
1198        }
1199
17111
6603
        $name =~ s/\A\s+//;
1200
1201        # Shorten over-long names (a hostile or corrupt database can give any
1202        # length), before trailing spaces and dots are removed, as shortening
1203        # may leave some at the end
1204
17111
8118
        $name = $self->_shorten_name($name);
1205
17111
7166
        $name =~ s/[\s.]+\z//;
1206
1207        # Avoid hidden files, Windows device names and empty names
1208
17111
5877
        $name =~ s/\A\./_/;
1209
17111
10861
        $name = "_$name" if $name =~ $RESERVED_NAME_RE;
1210
17111
6755
        $name = $UNNAMED unless length $name;
1211
1212        # Find the smallest free suffix (_2, _3, ...).  Start from where this
1213        # name's last search ended rather than from 2: every suffix below that
1214        # point was taken then and is still taken (names are never released
1215        # during a run), so the answer is the same, but N tables with the same
1216        # name cost O(N) in total instead of O(N squared).
1217
17111
6311
        my $used = $self->{used_names};
1218
17111
6260
        my $base = lc $name;
1219
17111
6159
        my $file = $name . $CSV_SUFFIX;
1220
17111
8033
        if(exists $used->{lc $file}) {
1221
11698
5971
                my $n = $self->{next_suffix}{$base} // 2;
1222
11698
6314
                $n++ while exists $used->{lc "${name}_$n$CSV_SUFFIX"};
1223
11698
4007
                $file = "${name}_$n$CSV_SUFFIX";
1224
11698
4618
                $self->{next_suffix}{$base} = $n + 1;
1225        }
1226
17111
8465
        $used->{lc $file} = 1;
1227
1228
17111
12908
        return $file;
1229
34
34
34
7843
18
100
}
1230
1231# _shorten_name
1232# Purpose:        Keep a file name within $MAX_NAME_CHARS characters and
1233#                 $MAX_NAME_BYTES bytes of UTF-8, without cutting a
1234#                 character, or a character from its accents (a grapheme,
1235#                 such as a joined emoji), in half.
1236# Entry Criteria: $name is a character string, or a byte string of valid
1237#                 UTF-8 (_csv_filename has replaced any other bytes).
1238# Exit Status:    Returns $name if it fits, otherwise its longest
1239#                 beginning that fits, in the same form (bytes or
1240#                 characters).
1241# Side Effects:   None.
1242sub _shorten_name :Private {
1243
17122
29688
        my ($self, $name) = @_;
1244
1245
17122
7236
        my $chars = utf8::is_utf8($name);
1246
17122
6107
        my $bytes = $name;
1247
17122
7113
        utf8::encode($bytes) if $chars;
1248
1249
17122
5781
        my $text = $bytes;
1250
17122
8748
        utf8::decode($text);
1251
17122
12862
        return $name if length($text) <= $MAX_NAME_CHARS && length($bytes) <= $MAX_NAME_BYTES;
1252
1253        # Whole graphemes, as many as fit both limits
1254
22
33
        my ($kept, $count, $size) = ('', 0, 0);
1255
22
435
        foreach my $grapheme ($text =~ /(\X)/g) {
1256
1271
414
                my $encoded = $grapheme;
1257
1271
599
                utf8::encode($encoded);
1258
1271
879
                last if $count + length($grapheme) > $MAX_NAME_CHARS
1259                        || $size + length($encoded) > $MAX_NAME_BYTES;
1260
1249
422
                $kept .= $grapheme;
1261
1249
398
                $count += length $grapheme;
1262
1249
417
                $size += length $encoded;
1263        }
1264
22
97
        utf8::encode($kept) unless $chars;
1265
22
24
        return $kept;
1266
34
34
34
6538
10
95
}
1267
1268# _dry_run
1269# Purpose:        Print the table -> file mapping without writing anything.
1270# Entry Criteria: $tables is the arrayref of selected tables.
1271# Exit Status:    Returns $self.
1272# Side Effects:   Prints to STDOUT; runs mdb-count when show_counts is on.
1273sub _dry_run :Private {
1274
66
868
        my ($self, $database, $tables) = @_;
1275
1276        # Only show the ROWS column when counts are actually available
1277
66
121
        my $counts = $self->{show_counts};
1278
66
225
        my $format = $counts
1279                ? "%-${TABLE_COLUMN_WIDTH}s %${ROWS_COLUMN_WIDTH}s  %s\n"
1280                : "%-${TABLE_COLUMN_WIDTH}s %s\n";
1281
66
146
162
3172
        my @header = map { $self->i18n($_) } ($counts ? qw(column_table column_rows column_output) : qw(column_table column_output));
1282
1283        # Underline the title to the length of whatever the translation is
1284
66
2179
        my $title = $self->i18n('dry_run_title');
1285
66
2938
        print "\n$title\n", '=' x length($title), "\n\n";
1286
66
345
        printf $format, @header;
1287
66
236
        print '-' x $RULE_WIDTH, "\n";
1288
1289
66
66
57
103
        foreach my $table (@{$tables}) {
1290
89
123
                my @row = ($table);
1291                # A count that cannot be had is shown as "?" (with a warning), so a
1292                # dry run still lists every table and succeeds
1293
89
149
                push @row, $self->_try_count_rows($database, $table) // $UNKNOWN_COUNT if $counts;
1294                # The table name comes from the database, so show it escaped (the
1295                # file name is already safe; escaping it too costs nothing)
1296
89
246
                $row[0] = $self->_printable($row[0]);
1297
89
199
                printf $format, @row, $self->_printable($self->_csv_filename($table));
1298        }
1299
66
143
        print "\n";
1300
1301
66
99
        return $self;
1302
34
34
34
7034
13
89
}
1303
1304# _warn
1305# Purpose:        Warn the user and record the same text in the log.
1306# Entry Criteria: $key is a catalog key; $args an optional i18n() hashref.
1307# Exit Status:    Returns $self.
1308# Side Effects:   carp()s; logs at warn level.
1309sub _warn :Private {
1310
110
775
        my ($self, $key, $args) = @_;
1311
1312
110
303
        $self->_carp_i18n($key, $args);
1313
110
181
        return $self->_log(warn => $key, $args);
1314
34
34
34
5407
13
77
}
1315
1316# _log
1317# Purpose:        Send a localised message to the logger, if there is one.
1318# Entry Criteria: $level is a logger method name (debug, info or warn).
1319# Exit Status:    Returns $self.
1320# Side Effects:   Calls $self->{logger}->$level().
1321sub _log :Private {
1322
1648
7228
        my ($self, $level, $key, $args) = @_;
1323
1324
1648
1732
        my $logger = $self->{logger} or return $self;
1325
1326        # Logging is secondary: a logger that dies (full disk, closed socket)
1327        # must not make a finished export look failed.  Say so once and stop
1328        # using it.
1329
174
136
        local $@;
1330        # Escaped, so a hostile table name cannot forge log lines (CR, LF)
1331        # or send escape sequences to whoever views the log
1332
174
174
169
143
523
13632
        if(!eval { $logger->$level($self->_printable($self->i18n($key, $args))); 1 }) {
1333
5
19
                my $error = $@;
1334
5
6
                chomp $error;
1335
5
4
                delete $self->{logger};
1336
5
10
                $self->_carp_i18n('log_failed', { params => [$error] });
1337        }
1338
174
166
        return $self;
1339
34
34
34
5824
14
84
}
1340
1341# _validation_reason
1342# Purpose:        Reduce a Params::Validate::Strict error to its meaning,
1343#                 e.g. "Parameter 'encoding' (latin1) must be one of utf8,
1344#                 utf8-bom, cp1252", dropping the module's own prefix
1345#                 ("Params::Validate::Strict line N: validate_strict: ")
1346#                 and the Perl location it appends.
1347# Entry Criteria: $error is the exception from validate_strict.
1348# Exit Status:    Returns a one-line string.
1349# Side Effects:   None.  A plain function, not a method.
1350sub _validation_reason :Private {
1351
92
28537
        my $error = shift;
1352
1353
92
51
        my $reason = "$error";
1354
92
131
        $reason =~ s/\AParams::Validate::Strict line \d+: //;
1355
92
78
        $reason =~ s/\Avalidate_strict: //;
1356
92
142
        $reason =~ s/ at \S+ line \d+\.?\n?\z//;
1357
92
50
        chomp $reason;
1358
92
117
        return $reason;
1359
34
34
34
5746
14
76
}
1360
1361# _untaint
1362# Purpose:        Mark a value that has already been validated as safe for
1363#                 taint mode.
1364# Entry Criteria: $value has been checked by the caller (see each call).
1365# Exit Status:    Returns the untainted value; returns $value unchanged if
1366#                 it is empty or contains a NUL (it will then fail safely
1367#                 at the operating system, still tainted).
1368# Side Effects:   None.  A plain function, not a method.
1369sub _untaint :Private {
1370
34650
66537
        my $value = shift;
1371
1372
34650
38432
        return ($value // '') =~ $UNTAINT_RE ? $1 : $value;
1373
34
34
34
4601
16
66
}
1374
1375# _os_error
1376# Purpose:        Get a readable OS error from an autodie exception or string.
1377# Entry Criteria: $error is whatever eval left in $@.
1378# Exit Status:    Returns a string.
1379# Side Effects:   None.  A plain function, not a method.
1380sub _os_error :Private {
1381
14
3329
        my $error = shift;
1382
1383        # autodie::exception keeps the original $! for us.  blessed(), not
1384        # ref(): asking an unblessed reference ->can() would die and hide
1385        # the real error.
1386
14
90
        my $text = (blessed($error) && $error->can('errno')) ? $error->errno() : "$error";
1387
14
22
        chomp $text;
1388
14
29
        return $text;
1389
34
34
34
5408
13
69
}
1390
13911;
1392