| File: | blib/lib/App/Access2CSV/Exporter.pm |
| Coverage: | 98.9% |
| line | stmt | bran | cond | sub | time | code |
|---|---|---|---|---|---|---|
| 1 | package App::Access2CSV::Exporter; | |||||
| 2 | ||||||
| 3 | 36 36 36 | 76133 14 170 | use strict; | |||
| 4 | 36 36 36 | 21 17 394 | use warnings; | |||
| 5 | 36 36 36 | 213 6215 42 | use autodie qw(:all); | |||
| 6 | ||||||
| 7 | 36 36 36 | 46742 14 511 | use Config; | |||
| 8 | ||||||
| 9 | # Sub::Private must be in enforce mode before it is loaded, so that | |||||
| 10 | # private methods still work through $self->method dispatch | |||||
| 11 | 36 | 178 | BEGIN { $Sub::Private::config{mode} = 'enforce' } | |||
| 12 | ||||||
| 13 | # Inherit i18n() and the protected _croak_i18n/_carp_i18n helpers | |||||
| 14 | 36 36 36 | 20 16 43 | use parent 'App::Access2CSV::I18N'; | |||
| 15 | ||||||
| 16 | 36 36 36 | 494 14 411 | use Encode qw(FB_CROAK find_encoding); | |||
| 17 | 36 36 36 | 29 13 324 | use File::Path qw(make_path); | |||
| 18 | 36 36 36 | 22 17 193 | use File::Spec; | |||
| 19 | 36 36 36 | 4698 72544 492 | use File::Temp; | |||
| 20 | 36 35 35 | 3852 9257 443 | use File::Which qw(which); | |||
| 21 | 35 34 34 | 4132 24619 469 | use IPC::Run3 qw(run3); | |||
| 22 | 34 34 34 | 45 12 242 | use Params::Get qw(get_params); | |||
| 23 | 34 34 34 | 23 18 195 | use Params::Validate::Strict qw(validate_strict); | |||
| 24 | 34 34 34 | 25 11 213 | use Readonly; | |||
| 25 | 34 34 34 | 26 11 189 | use Return::Set qw(set_return); | |||
| 26 | 34 34 34 | 16 19 195 | use Scalar::Util qw(blessed); | |||
| 27 | 34 34 34 | 13 20 60 | use Sub::Private; | |||
| 28 | 34 34 34 | 1724 15 35 | use Sub::Protected; | |||
| 29 | ||||||
| 30 | our $VERSION = '0.001.1'; | |||||
| 31 | ||||||
| 32 | # Stop Carp from reporting errors against the access-control wrappers | |||||
| 33 | our @CARP_NOT = qw(Sub::Private Sub::Protected App::Access2CSV::I18N); | |||||
| 34 | ||||||
| 35 | # Exit statuses returned by run() | |||||
| 36 | Readonly::Scalar my $EXIT_OK => 0; | |||||
| 37 | Readonly::Scalar my $EXIT_FAILURE => 1; | |||||
| 38 | ||||||
| 39 | # The mdbtools programs; mdb-count is only needed for --show-counts | |||||
| 40 | Readonly::Scalar my $MDB_TABLES => 'mdb-tables'; | |||||
| 41 | Readonly::Scalar my $MDB_EXPORT => 'mdb-export'; | |||||
| 42 | Readonly::Scalar my $MDB_COUNT => 'mdb-count'; | |||||
| 43 | Readonly::Array my @REQUIRED_PROGRAMS => ($MDB_TABLES, $MDB_EXPORT); | |||||
| 44 | ||||||
| 45 | # Output encodings accepted by --encoding | |||||
| 46 | Readonly::Scalar my $ENC_UTF8 => 'utf8'; | |||||
| 47 | Readonly::Scalar my $ENC_UTF8_BOM => 'utf8-bom'; | |||||
| 48 | Readonly::Scalar my $ENC_CP1252 => 'cp1252'; | |||||
| 49 | Readonly::Array my @ENCODINGS => ($ENC_UTF8, $ENC_UTF8_BOM, $ENC_CP1252); | |||||
| 50 | ||||||
| 51 | # Taint mode: a value is untainted only after it has been validated, by | |||||
| 52 | # capturing it with this pattern (anything non-empty without a NUL byte: | |||||
| 53 | # a NUL cannot be passed to the operating system at all) | |||||
| 54 | Readonly::Scalar my $UNTAINT_RE => qr/\A([^\x00]+)\z/s; | |||||
| 55 | ||||||
| 56 | # Environment variables that can change how a program is started (see | |||||
| 57 | # perlsec); removed for the mdbtools processes | |||||
| 58 | Readonly::Array my @UNSAFE_ENV => qw(IFS CDPATH ENV BASH_ENV); | |||||
| 59 | ||||||
| 60 | # Encoding objects, looked up once: calling Encode::decode/encode by name | |||||
| 61 | # repeats the lookup for every line, which made conversion about 4 times | |||||
| 62 | # slower on large tables. (Plain lexicals, not Readonly: Readonly's deep | |||||
| 63 | # copy could interfere with the objects' internals.) | |||||
| 64 | my $UTF8_CODEC = find_encoding('UTF-8'); | |||||
| 65 | my $CP1252_CODEC = find_encoding('cp1252'); | |||||
| 66 | ||||||
| 67 | # Byte order mark written at the start of utf8-bom files (for Excel) | |||||
| 68 | Readonly::Scalar my $UTF8_BOM => "\xEF\xBB\xBF"; | |||||
| 69 | ||||||
| 70 | # Tables Access creates for itself: MSys*, USys* and ~temporary objects | |||||
| 71 | Readonly::Scalar my $SYSTEM_TABLE_RE => qr/\A(?:MSys|USys|~)/i; | |||||
| 72 | ||||||
| 73 | # Characters that are illegal in a file name on at least one common OS | |||||
| 74 | Readonly::Scalar my $UNSAFE_CHARS_RE => qr/[<>:"\/\\|?*\x00-\x1F\x7F]/; | |||||
| 75 | ||||||
| 76 | # Invisible text-direction controls (LRM, RLM, LRE, RLE, PDF, LRO, RLO, | |||||
| 77 | # LRI, RLI, FSI, PDI). They can make a file name display as something | |||||
| 78 | # else ("report<RLO>vsc.exe.csv"), so they are unsafe like other control | |||||
| 79 | # characters. Matched both as Perl characters and as UTF-8 bytes, since | |||||
| 80 | # table names from mdbtools arrive as bytes. | |||||
| 81 | Readonly::Scalar my $BIDI_CONTROLS_RE => qr/ | |||||
| 82 | [\x{200E}\x{200F}\x{202A}-\x{202E}\x{2066}-\x{2069}] # as characters | |||||
| 83 | | \xE2 \x80 [\x8E\x8F\xAA-\xAE] # as UTF-8: marks, embeddings, overrides | |||||
| 84 | | \xE2 \x81 [\xA6-\xA9] # as UTF-8: isolates | |||||
| 85 | /x; | |||||
| 86 | ||||||
| 87 | # Device names Windows reserves whatever the extension (CON.csv is illegal) | |||||
| 88 | Readonly::Scalar my $RESERVED_NAME_RE => qr/\A(?:CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])\z/i; | |||||
| 89 | ||||||
| 90 | # Name used when sanitising leaves nothing, and the extension we write | |||||
| 91 | Readonly::Scalar my $UNNAMED => 'unnamed'; | |||||
| 92 | Readonly::Scalar my $CSV_SUFFIX => '.csv'; | |||||
| 93 | ||||||
| 94 | # Longest file name (before any _2 suffix and ".csv"). Access allows 64 | |||||
| 95 | # characters, so a longer name comes from a damaged or hostile database. | |||||
| 96 | # 64 characters can still be 256 bytes of UTF-8 (emoji), and most file | |||||
| 97 | # systems allow 255 bytes per name, so there is a byte limit too, leaving | |||||
| 98 | # room for "_" plus a collision number and ".csv" | |||||
| 99 | Readonly::Scalar my $MAX_NAME_CHARS => 64; | |||||
| 100 | Readonly::Scalar my $MAX_NAME_BYTES => 240; | |||||
| 101 | ||||||
| 102 | # Ends option parsing in mdbtools, so a table or database name that | |||||
| 103 | # starts with "-" is never taken for an option (glib parses options | |||||
| 104 | # anywhere on the command line, not only before the file name) | |||||
| 105 | Readonly::Scalar my $END_OF_OPTIONS => '--'; | |||||
| 106 | ||||||
| 107 | # Temporary files are hidden and live next to the target for atomic rename | |||||
| 108 | Readonly::Scalar my $TEMP_TEMPLATE => '.access2csv-XXXXXX'; | |||||
| 109 | ||||||
| 110 | # How much of an unreadable mdb-count answer to quote in the warning | |||||
| 111 | Readonly::Scalar my $COUNT_SHOWN => 40; | |||||
| 112 | ||||||
| 113 | # Shown in the dry run's ROWS column when a table could not be counted | |||||
| 114 | Readonly::Scalar my $UNKNOWN_COUNT => '?'; | |||||
| 115 | ||||||
| 116 | # Dry-run table layout | |||||
| 117 | Readonly::Scalar my $TABLE_COLUMN_WIDTH => 40; | |||||
| 118 | Readonly::Scalar my $ROWS_COLUMN_WIDTH => 10; | |||||
| 119 | Readonly::Scalar my $RULE_WIDTH => 70; | |||||
| 120 | ||||||
| 121 | # Return schema of run(), shared by its two exits | |||||
| 122 | Readonly::Hash my %RUN_STATUS_SCHEMA => (type => 'integer', min => $EXIT_OK, max => $EXIT_FAILURE); | |||||
| 123 | ||||||
| 124 | # Mode bits for new files before the umask is applied | |||||
| 125 | Readonly::Scalar my $FILE_MODE => oct('666'); | |||||
| 126 | ||||||
| 127 | # Default settings; the flat scalar layout is compatible with Object::Configure | |||||
| 128 | Readonly::Hash my %DEFAULTS => ( | |||||
| 129 | output_dir => File::Spec->curdir(), | |||||
| 130 | overwrite => 0, | |||||
| 131 | verbose => 0, | |||||
| 132 | dry_run => 0, | |||||
| 133 | show_counts => 0, | |||||
| 134 | progress => 1, | |||||
| 135 | encoding => $ENC_UTF8, | |||||
| 136 | ); | |||||
| 137 | ||||||
| 138 | # Constructor argument schema, shared by new() and the POD | |||||
| 139 | Readonly::Hash my %NEW_SCHEMA => ( | |||||
| 140 | output_dir => { type => 'string', min => 1, optional => 1 }, | |||||
| 141 | tables => { type => 'arrayref', element_type => 'string', optional => 1 }, | |||||
| 142 | overwrite => { type => 'boolean', optional => 1 }, | |||||
| 143 | verbose => { type => 'boolean', optional => 1 }, | |||||
| 144 | dry_run => { type => 'boolean', optional => 1 }, | |||||
| 145 | show_counts => { type => 'boolean', optional => 1 }, | |||||
| 146 | progress => { type => 'boolean', optional => 1 }, | |||||
| 147 | encoding => { type => 'string', memberof => [@ENCODINGS], optional => 1 }, | |||||
| 148 | logger => { type => 'object', can => ['debug', 'info', 'warn'], optional => 1 }, | |||||
| 149 | language => { type => 'string', optional => 1 }, | |||||
| 150 | ); | |||||
| 151 | ||||||
| 152 | =encoding utf8 | |||||
| 153 | ||||||
| 154 - 440 | =head1 NAME
App::Access2CSV::Exporter - Export the tables of a Microsoft Access database to CSV files
=head1 VERSION
Version 0.001.1
=head1 SYNOPSIS
use App::Access2CSV::Exporter;
# 1. The simplest case: every table, into the current directory
my $exporter = App::Access2CSV::Exporter->new();
my $status = $exporter->run('shop.accdb'); # 0 = all OK, 1 = some failed
# 2. Some tables, into a directory, for Excel, replacing old files
my $exporter = App::Access2CSV::Exporter->new(
output_dir => 'exports',
tables => ['Customers', 'Orders'],
encoding => 'utf8-bom',
overwrite => 1,
);
$exporter->run('shop.accdb');
# 3. Only look: print the table list and row counts, write nothing
App::Access2CSV::Exporter->new(dry_run => 1, show_counts => 1)->run('shop.accdb');
# 4. Inside a larger program: no progress lines, a log, and full
# error handling
use Log::Abstraction;
my $exporter = App::Access2CSV::Exporter->new({
output_dir => '/srv/exports',
progress => 0,
logger => Log::Abstraction->new(logger => '/var/log/export.log'),
});
my $status = eval { $exporter->run('/data/shop.accdb') };
if(!defined $status) {
die "Nothing was exported: $@"; # for example, the file is missing
} elsif($status == 1) {
warn "Some tables were not exported; see the log\n";
}
=head1 DESCRIPTION
This module does the real work of the C<access2csv> program. It writes
one CSV file for each table of a Microsoft Access database.
It runs three programs from the B<mdbtools> package: C<mdb-tables> (to
list the tables), C<mdb-export> (to get each table as CSV) and, only when
row counts are wanted, C<mdb-count>. They must be in your C<PATH>.
Access's own internal tables (names starting with C<MSys>, C<USys> or
C<~>) are skipped.
Each file is first written to a hidden temporary file in the output
directory, and renamed to its real name only when it is complete. So a
failed export never leaves a half-written CSV file, and an old file is
only replaced by a complete new one. New files get the usual
permissions (0666 minus your umask).
An exporter can be used for more than one C<run>. Each C<run> starts
again with the same file names, so running twice gives the same files.
The mdbtools programs are looked up only in absolute C<PATH> directory, so
a program planted in the current directory is never run, and they are
started with a cleaned environment (see L<App::Access2CSV/SECURITY>).
The module works under taint mode (C<perl -T>). Table names are
printed and logged with control characters escaped, so a hostile name
cannot send escape sequences to your terminal.
Table names and the database path are handed to mdbtools as separate
arguments, never through a shell, and after a C<--> marker. So names
containing shell characters (C<; | E<gt> $( )>), spaces or newlines, or
starting with C<->, are always treated as names, never as commands or
options. A table name can never place a file outside the output
directory: C</> and C<\> are replaced, and names cannot start with a dot.
An existing entry at the target name - including a symbolic link, even a
broken one - counts as "already exists". With C<overwrite>, the link
itself is replaced; the file it pointed to is never written.
The rules for file names are described in
L<App::Access2CSV/How the CSV files are named>.
=head1 ENCODING
=over 4
=item * B<CSV data.> mdbtools gives UTF-8. With C<encoding> set to
C<utf8> or C<utf8-bom> the bytes are copied exactly, so every character,
including emoji and non-Latin scripts, is kept. C<utf8-bom> also writes
the three-byte UTF-8 "byte order mark" first, which Microsoft Excel
needs. With C<cp1252>, each line is converted to Windows-1252; if a line
has a character that Windows-1252 does not have (for example Greek,
Chinese or an emoji), that table fails and nothing is written for it.
=item * B<Database path and output_dir.> These are passed to the operating
system unchanged. Give them as byte strings (the form you get from
C<@ARGV> or C<readdir>). Non-ASCII names work on systems whose file names
are UTF-8, such as Linux and macOS.
=item * B<Table names> (in C<tables>). They are compared with the names
that C<mdb-tables> prints, which are UTF-8 bytes. So give UTF-8 byte
strings, not decoded Perl character strings. If you have a decoded
string, use C<Encode::encode('UTF-8', $name)> first. The CSV file name is
made from the same bytes, so non-ASCII names and emoji are kept.
=item * B<Messages.> All messages are plain ASCII English.
=back
=head1 COMMON PITFALLS
=over 4
=item * B<undef means "use the default", not "false".> In C<new>,
C<< overwrite => undef >> is the same as not giving C<overwrite> at all.
To switch something off, give C<0>.
=item * B<An empty table list exports nothing.> C<< tables => undef >>
(or no C<tables>) means "all tables". C<< tables => [] >> means "no
tables": nothing is exported, and C<run> returns 0.
=item * B<Table names are case-sensitive.> C<'orders'> does not match the
table C<Orders>. Names that do not match any table give a warning.
=item * B<A failing logger does not stop the export.> If the logger dies
(for example, its disk is full), C<run> warns once with "Cannot write to
the log", stops logging for this exporter, and carries on exporting.
=item * B<run can croak.> C<run> returns 1 when some tables fail, but it
croaks (throws an exception) when nothing can be exported at all: the
database is missing or unreadable, mdbtools is not installed, or the
output directory cannot be created. Wrap C<run> in C<eval> if your program
must keep going.
=item * B<run may change the show_counts setting.> If C<show_counts> is
on but C<mdb-count> cannot be found, C<run> warns and switches
C<show_counts> off for this exporter.
=item * B<Settings are copied, not shared.> C<new> makes its own copy of
the C<tables> list; changing your array later has no effect. Settings are
not merged in depth: a new C<tables> list replaces the default completely.
=item * B<Warnings go through carp.> Failed tables and unknown table names
are reported with C<carp>, so they appear on standard error (or in your
C<$SIG{__WARN__}> handler) even when a logger is given.
=item * B<Load the module with use, not require.> Protection of the
private methods is set up at compile time. After a run-time
C<require> Perl prints "Too late to run CHECK block" and the protection is
missing.
=back
=head1 METHODS
=head2 new
=head3 Purpose
Make a new exporter with your settings. Nothing is checked on disk yet.
=head3 Arguments
Named arguments, either as a list or as one hash reference. All of them
are optional. An argument whose value is C<undef> is ignored, so its
default is used.
=over 4
=item C<output_dir> - the directory for the CSV files. Default: the current directory.
=item C<tables> - an array reference of table names to export. Default:
all tables. An empty array means no tables.
=item C<overwrite> - true to replace CSV files that already exist. Default: false.
=item C<verbose> - true to log extra detail. Default: false.
=item C<dry_run> - true to only print what would be written. Default: false.
=item C<show_counts> - true to report row counts (needs C<mdb-count>). Default: false.
=item C<progress> - true to print C<[n/total] table> lines to standard
error. Default: true.
=item C<encoding> - C<utf8>, C<utf8-bom> or C<cp1252>. Default: C<utf8>.
=item C<logger> - an object with C<debug>, C<info> and C<warn> methods,
such as a L<Log::Abstraction> object. Default: no logging.
=item C<language> - a language code such as C<en> for messages. Default:
taken from the locale (see L<App::Access2CSV::I18N>).
=back
=head3 Returns
A new C<App::Access2CSV::Exporter> object.
=head3 Side Effects
None. Your C<$@>, C<$!> and C<$_> are left as they were.
=head3 Usage
my $exporter = App::Access2CSV::Exporter->new({ dry_run => 1 });
=head3 EXAMPLE
# Export two tables as Windows-1252, replacing older files, with a log
my $exporter = App::Access2CSV::Exporter->new(
output_dir => 'out',
tables => ['Customers', 'Orders'],
encoding => 'cp1252',
overwrite => 1,
logger => Log::Abstraction->new(logger => 'export.log'),
);
=head3 API SPECIFICATION
=head4 Input
{
output_dir => { type => 'string', min => 1, optional => 1 },
tables => { type => 'arrayref', element_type => 'string', optional => 1 },
overwrite => { type => 'boolean', optional => 1 },
verbose => { type => 'boolean', optional => 1 },
dry_run => { type => 'boolean', optional => 1 },
show_counts => { type => 'boolean', optional => 1 },
progress => { type => 'boolean', optional => 1 },
encoding => { type => 'string', memberof => ['utf8', 'utf8-bom', 'cp1252'], optional => 1 },
logger => { type => 'object', can => ['debug', 'info', 'warn'], optional => 1 },
language => { type => 'string', optional => 1 },
}
Valid and invalid values (tested in F<t/domain.t>):
output_dir valid: any path of 1 character or more ("0" is valid),
including non-ASCII names given as UTF-8 bytes
invalid: "" (below the minimum), references
tables valid: undef (all tables), [] (no tables), one name,
many names; names are matched byte for byte
invalid: anything but an array reference; elements
that are references
booleans valid: exactly 1 true TRUE yes on / 0 false FALSE no off
(overwrite, invalid: everything else, e.g. "", 2, -1, "True", "Yes",
verbose, "0.0", " 1"
dry_run, show_counts, progress)
encoding valid: exactly utf8, utf8-bom, cp1252
invalid: other spellings (UTF8, utf-8, CP1252, " utf8")
logger valid: an object with debug, info and warn methods
invalid: an object missing any of them, a plain hash,
a string
language valid: any string; "" means "use the environment"
invalid: references
=head4 Output
{
type => 'object',
isa => 'App::Access2CSV::Exporter',
}
=head3 MESSAGES
All are fatal, and read "Invalid setting: REASON". The REASON part
comes from L<Params::Validate::Strict> and is not translated; for example:
+--------------------------------------+------------------------------+-----------------------------+
| Message | Meaning | What to do |
+--------------------------------------+------------------------------+-----------------------------+
| Invalid setting: Unknown parameter | X is not a known setting | Remove X, or fix its |
| 'X' | | spelling |
| Invalid setting: Parameter | This encoding is not | Use utf8, utf8-bom or |
| 'encoding' (X) must be one of utf8, | supported | cp1252 |
| utf8-bom, cp1252 | | |
| Invalid setting: Parameter 'logger' | logger is not an object | Give a logger object |
| must be an object | | |
| Invalid setting: Parameter 'tables' | tables is not an array | Give an array reference |
| must be ... | reference | |
+--------------------------------------+------------------------------+-----------------------------+
=cut | |||||
| 441 | ||||||
| 442 | sub new { | |||||
| 443 | 1809 | 4398126 | my $class = shift; | |||
| 444 | ||||||
| 445 | # Validation uses eval internally; the caller's $@ must survive | |||||
| 446 | 1809 | 794 | local $@; | |||
| 447 | ||||||
| 448 | # Drop undefined values so that "not given" means "use the default" | |||||
| 449 | 1809 | 2104 | my $args = get_params(undef, \@_) || {}; | |||
| 450 | 1808 2860 3117 1808 | 13152 1736 1490 1075 | my %given = map { $_ => $args->{$_} } grep { defined $args->{$_} } keys %{$args}; | |||
| 451 | ||||||
| 452 | # A bad setting is reported in plain words ("Invalid setting: ..."), | |||||
| 453 | # not with Params::Validate::Strict's internal prefix and location | |||||
| 454 | 1808 1808 | 910 1573 | my $params = eval { validate_strict(schema => { %NEW_SCHEMA }, input => \%given) } | |||
| 455 | or $class->_croak_i18n('invalid_setting', { params => [_validation_reason($@)] }); | |||||
| 456 | ||||||
| 457 | # Copy the table list so later changes by the caller cannot affect us | |||||
| 458 | 1716 46 | 473168 104 | $params->{tables} = [ @{ $params->{tables} } ] if $params->{tables}; | |||
| 459 | ||||||
| 460 | # The output directory is the invoking user's own choice; it is only used | |||||
| 461 | # as a directory name, so it is untainted here (see _untaint) | |||||
| 462 | 1716 | 1389 | $params->{output_dir} = _untaint($params->{output_dir}) if defined $params->{output_dir}; | |||
| 463 | ||||||
| 464 | 1716 1716 | 1239 10857 | my $self = bless { %DEFAULTS, %{$params}, used_names => {}, next_suffix => {}, programs => {} }, $class; | |||
| 465 | 1716 | 14000 | return set_return($self, { type => 'object' }); | |||
| 466 | } | |||||
| 467 | ||||||
| 468 - 682 | =head2 run
=head3 Purpose
Export the selected tables of one database to CSV files. In dry-run mode,
only print what would be exported.
=head3 Arguments
=over 4
=item C<database> (string, required) - the path of the C<.mdb> or C<.accdb> file.
The name is taken literally: C<-> is a file called C<->. (Reading from
standard input is a feature of the command-line program; see
L<App::Access2CSV/Reading the database from standard input>.)
=back
You can give it on its own, C<< $exporter->run('shop.accdb') >>, or as a
hash reference, C<< $exporter->run({ database => 'shop.accdb' }) >>.
=head3 Returns
C<0> if every selected table was exported, or in dry-run mode.
C<1> if at least one table was not exported (the others were).
=head3 Side Effects
=over 4
=item * Creates the output directory if needed (not in dry-run mode, and
not when no table is selected).
=item * Writes one CSV file per table (not in dry-run mode).
=item * Prints progress lines to standard error, if C<progress> is on.
=item * Prints the dry-run list to standard output, in dry-run mode.
=item * Sends messages to the logger, if there is one.
=item * Warns (with C<carp>) about each table that failed, about unknown
names in C<tables>, and about a missing C<mdb-count>.
=item * Croaks, before writing anything, if the database cannot be read, a
needed mdbtools program is missing, C<mdb-tables> fails, or the output
directory cannot be created.
=item * Switches C<show_counts> off for this exporter if C<mdb-count> is
missing.
=item * While it runs, handles the signals INT, QUIT, TERM and HUP (only
those you have not set a handler for yourself; they are restored when
C<run> returns). Any of them stops the run: the table being exported is
discarded - its temporary file deleted, any old CSV file left as it was -
no further table is started, and C<run> croaks. Pressing Ctrl-C, which
also stops the mdbtools program, has the same effect.
=item * Leaves your C<$@>, C<$!>, C<$?>, C<$_>, C<$.> and any pending C<alarm>
as they were (except that a croak sets C<$@> in your C<eval>, as usual).
=back
=head3 Usage
exit $exporter->run('shop.accdb');
=head3 EXAMPLE
my $exporter = App::Access2CSV::Exporter->new(output_dir => 'out');
# eval catches the fatal errors; the return value covers the rest
my $status = eval { $exporter->run('shop.accdb') };
if(!defined $status) {
print STDERR "Nothing was exported: $@";
} elsif($status) {
print STDERR "Some tables failed; see the warnings above\n";
} else {
print "Done\n";
}
=head3 API SPECIFICATION
=head4 Input
{
database => {
type => 'string',
min => 1,
optional => 0,
},
}
Valid and invalid values (tested in F<t/domain.t>):
database valid: a readable regular file
invalid: "" (below the 1-character minimum), undef,
a missing file, a directory, a device or FIFO,
an unreadable file
edges: each part of the path may be up to 255 bytes;
256 gives "File name too long"
The table names that mdbtools reports are data, not arguments, but they
have limits of their own:
length a CSV file name is the table name plus ".csv". Access
allows at most 64 characters, so longer names (only from
a damaged or hostile database) are shortened to 64
characters, and also to at most 240 bytes of UTF-8 (64
emoji would be 256 bytes; most file systems allow 255).
A name is never cut inside a character, nor between a
letter and its accents or inside a joined emoji.
Shortened names that clash are numbered as usual.
characters non-ASCII letters, emoji, joined emoji, combining marks
and right-to-left text are kept byte for byte.
Characters that are unsafe in file names - including
invisible text-direction controls such as U+202E - are
replaced by "_". So is each byte that is not part of
valid UTF-8 (only a damaged database has them; macOS
refuses such file names).
collisions the first name has no suffix, then _2, _3, ... _10 ...
cp1252 U+00FF and the Euro sign convert; U+0100 and above
(except the few Windows-1252 symbols), the C1 controls
U+0080-U+009F and emoji make the table fail.
=head4 Output
{
type => 'integer',
min => 0,
max => 1,
}
=head3 MESSAGES
"fatal" means C<run> croaks and nothing is exported. "per table" means
only that table fails; C<run> warns, logs, and carries on.
+-----------------------------------------+------------------------------+-------------------------------+
| Message | Meaning | What to do |
+-----------------------------------------+------------------------------+-------------------------------+
| Interrupted by SIGx: stopped, and the | Ctrl-C, Ctrl-\\, kill or a | Run again; tables finished |
| table being exported was discarded | closed terminal stopped the | before the interruption are |
| (fatal) | run | complete |
| run() must be called on an object | run was called on the class | Call new() first, then run() |
| created by new() (fatal) | or on something that is not | on the object it returns |
| | an exporter | |
| Cannot read database F: E (fatal) | F does not exist, or cannot | Check the path |
| | be reached; E is the reason | |
| | from the operating system | |
| Database F is not a regular file (fatal)| F is a directory or a device | Give the database file |
| Database F is not readable (fatal) | No permission to read F | Fix the permissions |
| | (never happens for root) | |
| Required program not found in PATH: P | mdbtools is not installed, | Install mdbtools, or fix PATH |
| (fatal) | or not in PATH | |
| mdb-tables failed with exit status N: E | mdbtools cannot read the | Check that F is a real Access |
| (fatal) | file | database |
| Cannot create output directory D: E | The directory cannot be made; | Check permissions and path |
| (fatal) | E is the reason for D itself | |
| | (e.g. "Not a directory" when | |
| | a file is in the way) | |
| Cannot count the rows of T: E (warning) | mdb-count failed for table T | The table is still exported |
| | (only with show_counts) | (dry run: count shown as "?") |
| ... mdb-count printed no number: "X" | mdb-count's answer was not | As above; X shows what it |
| | just a number | printed |
| Tables not found in database: T | Names in tables are not in | Check spelling and case |
| (warning) | the database | |
| mdb-count not found in PATH; row counts | show_counts is on, but | Install mdb-count, or turn |
| are unavailable (warning) | mdb-count is missing | show_counts off |
| FAILED: T: E (warning, logged) | Table T was not exported, | See E, one of the messages |
| | because of E | below |
| Output file already exists: F (use | F exists (a symbolic link, | Set overwrite, or use another |
| --overwrite to replace it) (per table) | even a broken one, counts) | output_dir |
| | and overwrite is off | |
| mdb-export failed with exit status N: E | mdbtools could not read this | Check the table in Access |
| (per table) | table | |
| P was killed by signal N (per table, | The program was stopped from | Check memory and system |
| or fatal for mdb-tables) | outside | limits |
| P could not be run: E (per table, or | The program was found but | Check its permissions and |
| fatal for mdb-tables) | could not be started | that it is a real program |
| Table T, line N: cannot be represented | A character is not in | Use utf8 or utf8-bom |
| in cp1252 (per table) | Windows-1252 | |
| Table T, line N: output of mdb-export | mdbtools gave bytes that are | Check the MDB_ICONV setting |
| is not valid UTF-8 (per table) | not UTF-8 | |
| Cannot write F: E (per table) | The file could not be | Check permissions and free |
| | written or renamed into place| disk space |
| Cannot write to the log: E (warning, | The logger failed. Exports | Check the log's disk or |
| once) | go on; logging stops | destination |
+-----------------------------------------+------------------------------+-------------------------------+
=head3 PSEUDOCODE
check the argument
stop (croak) unless the database is a readable file
find mdb-tables and mdb-export (croak if missing),
and mdb-count if row counts are wanted (warn if missing)
forget the file names given out by any earlier run
tables := the sorted user tables, filtered by "tables"
(warn about names that are not found)
if dry run:
print the table -> file list (row count "?" with a warning
if a count fails)
return 0
if there are tables to export:
create the output directory (croak if that fails)
for each table:
print "[n/total] table" if progress is on
try to export the table
if that failed: warn, log, and count the failure
(a failed row count after the file is in place is only a
warning; the table still counts as exported)
log the summary
return 1 if any table failed, else 0
=cut | |||||
| 683 | ||||||
| 684 | sub run { | |||||
| 685 | 401 | 68098 | my $self = shift; | |||
| 686 | ||||||
| 687 | # State machine guard: run() is only a transition out of READY, which | |||||
| 688 | # only new() can create. Refuse anything else before doing any work. | |||||
| 689 | # (Reported through the class: $self may not even be an object.) | |||||
| 690 | 401 | 1566 | blessed($self) && $self->isa(__PACKAGE__) or __PACKAGE__->_croak_i18n('needs_object'); | |||
| 691 | ||||||
| 692 | # File tests, evals and child processes below would otherwise leave | |||||
| 693 | # their marks in the caller's $@ and $! | |||||
| 694 | 397 | 1771 | local ($@, $!); | |||
| 695 | ||||||
| 696 | # An undef database is a missing one, not a file called "". Work on a | |||||
| 697 | # copy: get_params hands back the caller's own hash when given one. | |||||
| 698 | # (Params::Get either dies or returns a hash reference - proved in | |||||
| 699 | # t/path.t - so no test of what it returned is needed.) | |||||
| 700 | 397 397 | 248 431 | my $input = { %{ get_params('database', \@_) } }; | |||
| 701 | 391 | 4077 | delete $input->{database} unless defined $input->{database}; | |||
| 702 | 391 | 642 | my $params = validate_strict( | |||
| 703 | schema => { database => { type => 'string', min => 1 } }, | |||||
| 704 | input => $input, | |||||
| 705 | ); | |||||
| 706 | 382 | 14100 | my $database = $params->{database}; | |||
| 707 | ||||||
| 708 | # Fail fast, before any output, on problems that affect every table | |||||
| 709 | 382 | 697 | $self->_check_database($database) | |||
| 710 | ->_verify_dependencies() | |||||
| 711 | ->_reset_names(); | |||||
| 712 | ||||||
| 713 | # Stopping part-way must behave like a failed transaction: the table | |||||
| 714 | # being exported is discarded (its temporary file deleted) and no | |||||
| 715 | # further table is started. Perl's default action for these signals | |||||
| 716 | # is to exit at once, skipping the clean-up, so while run() is active | |||||
| 717 | # they raise an exception instead. A handler the caller has set is | |||||
| 718 | # left alone; everything is restored when run() returns. | |||||
| 719 | 312 | 301 | local $self->{interrupted}; | |||
| 720 | 312 312 | 206 513 | my @ours = @{ $self->_interrupt_signals() }; | |||
| 721 | local @SIG{@ours} = (sub { | |||||
| 722 | 1 | 32725 | $self->{interrupted} = $_[0]; | |||
| 723 | 1 | 14 | die $self->_printable($self->i18n('interrupted', { params => [$_[0]] })), "\n"; | |||
| 724 | 312 | 3863 | }) x @ours; | |||
| 725 | ||||||
| 726 | # Premise: the database is now known to be a readable regular file, and | |||||
| 727 | # it is only ever passed to mdbtools as one list argument after "--". | |||||
| 728 | # Conclusion: it is safe to untaint. | |||||
| 729 | 312 | 351 | $database = _untaint($database); | |||
| 730 | ||||||
| 731 | 312 | 538 | my $tables = $self->_select_tables($self->_get_tables($database)); | |||
| 732 | ||||||
| 733 | # Guard clause: a dry run must not touch the file system, so it leaves | |||||
| 734 | # before mkdir. Premise: _dry_run writes nothing that can fail a table. | |||||
| 735 | # Conclusion: a dry run always succeeds. | |||||
| 736 | 296 | 609 | if($self->{dry_run}) { | |||
| 737 | 63 | 226 | $self->_dry_run($database, $tables); | |||
| 738 | 63 | 289 | return set_return($EXIT_OK, { %RUN_STATUS_SCHEMA }); | |||
| 739 | } | |||||
| 740 | ||||||
| 741 | # The output directory is only made when there is something to put in it; | |||||
| 742 | # with no tables selected the run goes straight to the summary | |||||
| 743 | 233 233 | 230 728 | my $failed = (@{$tables} ? $self->_make_output_dir() : $self)->_export_all($database, $tables); | |||
| 744 | 224 | 1462 | return set_return($failed ? $EXIT_FAILURE : $EXIT_OK, { %RUN_STATUS_SCHEMA }); | |||
| 745 | } | |||||
| 746 | ||||||
| 747 | # _check_database | |||||
| 748 | # Purpose: Make sure the database is a readable regular file. | |||||
| 749 | # Entry Criteria: $database is a defined, non-empty path. | |||||
| 750 | # Exit Status: Returns $self for chaining; croaks otherwise. | |||||
| 751 | # Side Effects: stat()s the file; sets $!. | |||||
| 752 | sub _check_database :Private { | |||||
| 753 | 386 | 4137 | my ($self, $database) = @_; | |||
| 754 | ||||||
| 755 | # The stat result is reused via "_" so the file is only examined once; | |||||
| 756 | # $! is captured straight away because later calls may overwrite it | |||||
| 757 | 386 | 1334 | if(!-e $database) { | |||
| 758 | 36 | 214 | $self->_croak_i18n('database_not_found', { params => [$database, "$!"] }); | |||
| 759 | } | |||||
| 760 | 350 | 370 | $self->_croak_i18n('database_not_file', { params => [$database] }) unless -f _; | |||
| 761 | 337 | 531 | $self->_croak_i18n('database_unreadable', { params => [$database] }) unless -r _; | |||
| 762 | ||||||
| 763 | 331 | 646 | return $self; | |||
| 764 | 34 34 34 | 13342 14 310 | } | |||
| 765 | ||||||
| 766 | # _verify_dependencies | |||||
| 767 | # Purpose: Locate the mdbtools programs in PATH. | |||||
| 768 | # Entry Criteria: None. | |||||
| 769 | # Exit Status: Returns $self; croaks if a required program is missing. | |||||
| 770 | # Side Effects: Sets $self->{programs}; may switch off show_counts (with a | |||||
| 771 | # warning) when mdb-count is unavailable; logs at debug level. | |||||
| 772 | sub _verify_dependencies :Private { | |||||
| 773 | 354 | 3303 | my $self = shift; | |||
| 774 | ||||||
| 775 | 354 | 199 | my %programs; | |||
| 776 | 354 | 823 | foreach my $program (@REQUIRED_PROGRAMS) { | |||
| 777 | 684 | 1710 | $programs{$program} = $self->_find_program($program) | |||
| 778 | or $self->_croak_i18n('program_missing', { params => [$program] }); | |||||
| 779 | } | |||||
| 780 | ||||||
| 781 | # mdb-count is only needed for row counts, so its absence is not fatal. | |||||
| 782 | # _find_program returns a path or false, so one branch decides both | |||||
| 783 | # "store it" and "switch counts off" (nothing is stored and removed). | |||||
| 784 | 315 | 668 | if($self->{show_counts}) { | |||
| 785 | 47 | 62 | if(my $path = $self->_find_program($MDB_COUNT)) { | |||
| 786 | 35 | 58 | $programs{$MDB_COUNT} = $path; | |||
| 787 | } else { | |||||
| 788 | 12 | 21 | $self->{show_counts} = 0; | |||
| 789 | 12 | 40 | $self->_warn('no_row_counter'); | |||
| 790 | } | |||||
| 791 | } | |||||
| 792 | ||||||
| 793 | 315 | 325 | $self->{programs} = \%programs; | |||
| 794 | 315 | 586 | return $self; | |||
| 795 | 34 34 34 | 7297 18 126 | } | |||
| 796 | ||||||
| 797 | # _find_program | |||||
| 798 | # Purpose: Look up one program in PATH and note where it was found. | |||||
| 799 | # Entry Criteria: $program is a bare program name. | |||||
| 800 | # Exit Status: Returns the full path, or undef if not found. | |||||
| 801 | # Side Effects: Logs the location at debug level when --verbose is on. | |||||
| 802 | sub _find_program :Private { | |||||
| 803 | 730 | 2729 | my ($self, $program) = @_; | |||
| 804 | ||||||
| 805 | # Only absolute paths are trusted. A relative entry in PATH (".", or | |||||
| 806 | # an empty one) would run whatever file of that name is in the current | |||||
| 807 | # directory - a classic way to plant a program. | |||||
| 808 | 730 715 | 2129 37691 | my ($path) = grep { defined && File::Spec->file_name_is_absolute($_) } which($program); | |||
| 809 | ||||||
| 810 | # An absolute path to an existing program: safe to untaint | |||||
| 811 | 730 | 937 | $path = _untaint($path) if defined $path; | |||
| 812 | 730 | 1184 | if($path && $self->{verbose}) { | |||
| 813 | 8 | 13 | $self->_log(debug => 'program_found', { params => [$program, $path] }); | |||
| 814 | } | |||||
| 815 | 730 | 1044 | return $path; | |||
| 816 | 34 34 34 | 5740 14 123 | } | |||
| 817 | ||||||
| 818 | # _reset_names | |||||
| 819 | # Purpose: Forget file names allocated by a previous run() so that | |||||
| 820 | # running the same exporter twice gives the same names. | |||||
| 821 | # Entry Criteria: None. | |||||
| 822 | # Exit Status: Returns $self. | |||||
| 823 | # Side Effects: Empties $self->{used_names}. | |||||
| 824 | sub _reset_names :Private { | |||||
| 825 | 312 | 1076 | my $self = shift; | |||
| 826 | ||||||
| 827 | 312 | 271 | $self->{used_names} = {}; | |||
| 828 | 312 | 291 | $self->{next_suffix} = {}; | |||
| 829 | 312 | 203 | return $self; | |||
| 830 | 34 34 34 | 5099 19 89 | } | |||
| 831 | ||||||
| 832 | # _get_tables | |||||
| 833 | # Purpose: List the user tables in the database. | |||||
| 834 | # Entry Criteria: _verify_dependencies() has run. | |||||
| 835 | # Exit Status: Returns an arrayref of table names, sorted; croaks if | |||||
| 836 | # mdb-tables fails. | |||||
| 837 | # Side Effects: Runs mdb-tables. | |||||
| 838 | sub _get_tables :Protected { | |||||
| 839 | 308 | 1394 | my ($self, $database) = @_; | |||
| 840 | ||||||
| 841 | # -1 puts one table per line, so names containing spaces survive; | |||||
| 842 | # "--" stops a database path starting with "-" being read as an option | |||||
| 843 | 308 | 299 | my $stdout = ''; | |||
| 844 | 308 | 662 | $self->_run_program($MDB_TABLES, ['-1', $END_OF_OPTIONS, $database], \$stdout); | |||
| 845 | ||||||
| 846 | # \r? copes with mdbtools builds that emit CRLF line endings; a program | |||||
| 847 | # that printed nothing may leave $stdout undefined | |||||
| 848 | # Table names are untainted: they are only used as one list argument | |||||
| 849 | # after "--", and in file names only after _csv_filename has made them | |||||
| 850 | # safe | |||||
| 851 | 292 1524 401541 | 62541 1339 194730 | my @tables = sort map { _untaint($_) } grep { length($_) && !$self->_is_system_table($_) } split /\r?\n/, $stdout // ''; | |||
| 852 | 292 | 6560 | return \@tables; | |||
| 853 | 34 34 34 | 5556 38 110 | } | |||
| 854 | ||||||
| 855 | # _is_system_table | |||||
| 856 | # Purpose: Decide whether a table is Access's own rather than the user's. | |||||
| 857 | # Entry Criteria: $table is a table name. | |||||
| 858 | # Exit Status: Returns 1 for system tables, 0 otherwise. | |||||
| 859 | # Side Effects: None. Protected so that subclasses can widen the filter. | |||||
| 860 | sub _is_system_table :Protected { | |||||
| 861 | 401550 | 732056 | my ($self, $table) = @_; | |||
| 862 | ||||||
| 863 | 401550 | 316337 | return ($table =~ $SYSTEM_TABLE_RE) ? 1 : 0; | |||
| 864 | 34 34 34 | 4671 8 113 | } | |||
| 865 | ||||||
| 866 | # _select_tables | |||||
| 867 | # Purpose: Apply the --table filter to the list of tables. | |||||
| 868 | # Entry Criteria: $tables is the arrayref from _get_tables(). | |||||
| 869 | # Exit Status: Returns an arrayref, in database (sorted) order. | |||||
| 870 | # Side Effects: Warns and logs about requested tables that do not exist. | |||||
| 871 | sub _select_tables :Private { | |||||
| 872 | 302 | 2025 | my ($self, $tables) = @_; | |||
| 873 | ||||||
| 874 | 302 | 793 | return $tables unless $self->{tables}; | |||
| 875 | ||||||
| 876 | # Matching is exact, as mdb-export itself is case-sensitive | |||||
| 877 | 35 87 35 | 59 166 54 | my %available = map { $_ => 1 } @{$tables}; | |||
| 878 | 35 1050 35 | 48 519 56 | my %wanted = map { $_ => 1 } @{ $self->{tables} }; | |||
| 879 | ||||||
| 880 | 35 51 | 85 120 | my @missing = sort grep { !$available{$_} } keys %wanted; | |||
| 881 | 35 | 61 | if(@missing) { | |||
| 882 | 12 | 65 | $self->_warn('unknown_tables', { params => [join(', ', @missing)], count => scalar(@missing) }); | |||
| 883 | } | |||||
| 884 | ||||||
| 885 | 35 87 35 | 42 102 34 | return [ grep { $wanted{$_} } @{$tables} ]; | |||
| 886 | 34 34 34 | 5859 17 99 | } | |||
| 887 | ||||||
| 888 | # _make_output_dir | |||||
| 889 | # Purpose: Create the output directory (and parents) if necessary. | |||||
| 890 | # Entry Criteria: Not in dry-run mode. | |||||
| 891 | # Exit Status: Returns $self; croaks if the directory cannot be created. | |||||
| 892 | # Side Effects: Creates directories. | |||||
| 893 | sub _make_output_dir :Private { | |||||
| 894 | 231 | 1787 | my $self = shift; | |||
| 895 | ||||||
| 896 | 231 | 384 | my $dir = $self->{output_dir}; | |||
| 897 | 231 | 1627 | return $self if -d $dir; | |||
| 898 | ||||||
| 899 | # Ask File::Path to report errors rather than carp/croak on its own, | |||||
| 900 | # so the message can be translated and names the directory we wanted | |||||
| 901 | 177 | 19158 | make_path($dir, { error => \my $errors }); | |||
| 902 | 177 177 | 263 892 | if(@{$errors} || !-d $dir) { | |||
| 903 | # File::Path may also report a parent (e.g. "File exists" for a | |||||
| 904 | # plain file in the way); the reason for $dir itself, or failing | |||||
| 905 | # that the last one, is what explains the failure | |||||
| 906 | 18 24 18 | 17 79 24 | my ($mine) = grep { exists $_->{$dir} } @{$errors}; | |||
| 907 | 18 6 2 | 34 5 3 | my $detail = $mine ? $mine->{$dir} : (@{$errors} ? (values %{ $errors->[-1] })[0] : undef); | |||
| 908 | 18 | 78 | $self->_croak_i18n('mkdir_failed', { params => [$dir, $detail || "$!"] }); | |||
| 909 | } | |||||
| 910 | 159 | 441 | return $self; | |||
| 911 | 34 34 34 | 5711 16 119 | } | |||
| 912 | ||||||
| 913 | # _export_all | |||||
| 914 | # Purpose: Export each table, carrying on past individual failures. | |||||
| 915 | # Entry Criteria: The output directory exists. | |||||
| 916 | # Exit Status: Returns the number of tables that failed. | |||||
| 917 | # Side Effects: Writes CSV files; prints progress; warns; logs. | |||||
| 918 | sub _export_all :Private { | |||||
| 919 | 229 | 1732 | my ($self, $database, $tables) = @_; | |||
| 920 | ||||||
| 921 | 229 229 | 160 245 | my $total = scalar @{$tables}; | |||
| 922 | 229 | 208 | my $failed = 0; | |||
| 923 | ||||||
| 924 | # eval below would otherwise overwrite the caller's $@ | |||||
| 925 | 229 | 160 | local $@; | |||
| 926 | ||||||
| 927 | # An index loop, not each(), which shares the array's iterator with the | |||||
| 928 | # caller and would silently skip tables if it was already part-way | |||||
| 929 | 229 229 | 204 349 | foreach my $index (0 .. $#{$tables}) { | |||
| 930 | 1391 | 993 | my $table = $tables->[$index]; | |||
| 931 | # Progress goes to STDERR so that STDOUT can be redirected cleanly | |||||
| 932 | 1391 | 1204 | if($self->{progress}) { | |||
| 933 | 53 | 307 | print STDERR $self->_printable($self->i18n('progress', { params => [$index + 1, $total, $table] })), "\n"; | |||
| 934 | } | |||||
| 935 | ||||||
| 936 | # One bad table should not stop the rest from being exported | |||||
| 937 | 1391 1391 1315 | 704 1634 18784 | next if eval { $self->_export_table($database, $table); 1 }; | |||
| 938 | ||||||
| 939 | # ... but an interruption stops them all: the failed table has been | |||||
| 940 | # discarded (its temporary file went with the exception), and no | |||||
| 941 | # further table is started | |||||
| 942 | 76 | 18284 | $self->_croak_i18n('interrupted', { params => [$self->{interrupted}] }) if $self->{interrupted}; | |||
| 943 | ||||||
| 944 | 74 | 138 | my $error = $@ || 'Unknown error'; | |||
| 945 | 74 | 87 | chomp $error; | |||
| 946 | 74 | 62 | ++$failed; | |||
| 947 | 74 | 225 | $self->_warn('export_failed', { params => [$table, $error] }); | |||
| 948 | } | |||||
| 949 | ||||||
| 950 | 227 | 759 | $self->_log(info => 'summary', { params => [$total, $failed], count => $total }); | |||
| 951 | 227 | 556 | return $failed; | |||
| 952 | 34 34 34 | 5976 29 91 | } | |||
| 953 | ||||||
| 954 | # _export_table | |||||
| 955 | # Purpose: Export one table to its CSV file. | |||||
| 956 | # Entry Criteria: _verify_dependencies() and _make_output_dir() have run. | |||||
| 957 | # Exit Status: Returns $self; croaks on any failure, leaving no partial file. | |||||
| 958 | # Side Effects: Runs mdb-export (and mdb-count); creates or replaces a file. | |||||
| 959 | sub _export_table :Protected { | |||||
| 960 | 1391 | 5165 | my ($self, $database, $table) = @_; | |||
| 961 | ||||||
| 962 | 1391 | 1616 | my $outfile = File::Spec->catfile($self->{output_dir}, $self->_csv_filename($table)); | |||
| 963 | ||||||
| 964 | # Check before exporting so that we do not waste time on a big table | |||||
| 965 | # -l as well as -e: a dangling symlink is an existing entry too, and | |||||
| 966 | # must not be silently replaced. The overwrite flag is tested first: | |||||
| 967 | # when it is set the answer is already known, so no file test is needed. | |||||
| 968 | 1391 | 11068 | if(!$self->{overwrite} && (-e $outfile || -l $outfile)) { | |||
| 969 | 24 | 115 | $self->_croak_i18n('output_exists', { params => [$outfile] }); | |||
| 970 | } | |||||
| 971 | ||||||
| 972 | # Write into a temporary file next to the target; it is deleted | |||||
| 973 | # automatically if anything below croaks | |||||
| 974 | 1367 | 2684 | my $tmp = File::Temp->new(DIR => $self->{output_dir}, TEMPLATE => $TEMP_TEMPLATE, UNLINK => 1); | |||
| 975 | 1365 | 129577 | binmode $tmp, ':raw'; | |||
| 976 | ||||||
| 977 | 1365 | 28273 | if($self->{encoding} eq $ENC_CP1252) { | |||
| 978 | 45 | 104 | $self->_export_transcoded($database, $table, $tmp); | |||
| 979 | } else { | |||||
| 980 | # The BOM must reach the file before mdb-export starts writing to | |||||
| 981 | # the same descriptor, hence the explicit flush | |||||
| 982 | 1320 19 | 996 72 | print {$tmp} $UTF8_BOM if $self->{encoding} eq $ENC_UTF8_BOM; | |||
| 983 | 1320 | 1692 | $tmp->flush() or $self->_croak_i18n('write_failed', { params => [$outfile, "$!"] }); | |||
| 984 | 1317 | 1596 | $self->_run_program($MDB_EXPORT, [$END_OF_OPTIONS, $database, $table], $tmp); | |||
| 985 | } | |||||
| 986 | ||||||
| 987 | 1320 | 4751 | $self->_install_file($tmp, $outfile); | |||
| 988 | ||||||
| 989 | # The file is now in place, so the table has been exported. Row counts | |||||
| 990 | # are optional extras: if counting fails it is only a warning (see | |||||
| 991 | # _try_count_rows), and the export is logged without a count. | |||||
| 992 | 1316 | 1668 | my $rows = $self->{show_counts} ? $self->_try_count_rows($database, $table) : undef; | |||
| 993 | 1316 | 977 | if(defined $rows) { | |||
| 994 | 31 | 271 | $self->_log(info => 'exported_rows', { params => [$table, $outfile, $rows], count => $rows }); | |||
| 995 | } else { | |||||
| 996 | 1285 | 1955 | $self->_log(info => 'exported', { params => [$table, $outfile] }); | |||
| 997 | } | |||||
| 998 | 1316 | 1737 | return $self; | |||
| 999 | 34 34 34 | 6435 9 99 | } | |||
| 1000 | ||||||
| 1001 | # _export_transcoded | |||||
| 1002 | # Purpose: Export a table and convert it from UTF-8 to Windows-1252. | |||||
| 1003 | # Entry Criteria: $out is an open, raw, writable filehandle. | |||||
| 1004 | # Exit Status: Returns $self; croaks on invalid UTF-8 or on a character | |||||
| 1005 | # that has no cp1252 equivalent (rather than silently | |||||
| 1006 | # replacing it with '?'). | |||||
| 1007 | # Side Effects: Runs mdb-export into a second temporary file. | |||||
| 1008 | sub _export_transcoded :Private { | |||||
| 1009 | 51 | 1251 | my ($self, $database, $table, $out) = @_; | |||
| 1010 | ||||||
| 1011 | # Reading the spool changes $. and the evals change $@; keep the | |||||
| 1012 | # caller's values | |||||
| 1013 | 51 | 69 | local $.; | |||
| 1014 | 51 | 42 | local $@; | |||
| 1015 | ||||||
| 1016 | # Spool to disk rather than memory, so huge tables do not exhaust RAM | |||||
| 1017 | 51 | 104 | my $spool = File::Temp->new(DIR => $self->{output_dir}, TEMPLATE => $TEMP_TEMPLATE, UNLINK => 1); | |||
| 1018 | 51 | 3796 | binmode $spool, ':raw'; | |||
| 1019 | 51 | 729 | $self->_run_program($MDB_EXPORT, [$END_OF_OPTIONS, $database, $table], $spool); | |||
| 1020 | 43 | 295 | seek $spool, 0, 0; | |||
| 1021 | ||||||
| 1022 | # Convert line by line; $. gives the user a line number to look at | |||||
| 1023 | 43 | 4534 | while(my $line = <$spool>) { | |||
| 1024 | 81 81 | 64 242 | my $chars = eval { $UTF8_CODEC->decode($line, FB_CROAK) }; | |||
| 1025 | 81 | 344 | $self->_croak_i18n('invalid_utf8', { params => [$table, $.] }) unless defined $chars; | |||
| 1026 | ||||||
| 1027 | 74 74 | 52 219 | my $bytes = eval { $CP1252_CODEC->encode($chars, FB_CROAK) }; | |||
| 1028 | 74 | 140 | $self->_croak_i18n('unmappable', { params => [$table, $., $ENC_CP1252] }) unless defined $bytes; | |||
| 1029 | ||||||
| 1030 | 63 63 | 41 228 | print {$out} $bytes; | |||
| 1031 | } | |||||
| 1032 | ||||||
| 1033 | # Close the spool explicitly once it has been read. (On a croak above, | |||||
| 1034 | # File::Temp's destructor closes and deletes it.) | |||||
| 1035 | 25 | 83 | close $spool; | |||
| 1036 | 25 | 956 | return $self; | |||
| 1037 | 34 34 34 | 6530 20 91 | } | |||
| 1038 | ||||||
| 1039 | # _install_file | |||||
| 1040 | # Purpose: Move a finished temporary file to its final name. | |||||
| 1041 | # Entry Criteria: $tmp is a File::Temp holding the complete CSV. | |||||
| 1042 | # Exit Status: Returns $self; croaks if the rename or chmod fails. | |||||
| 1043 | # Side Effects: Replaces $outfile; the temporary file is no longer | |||||
| 1044 | # auto-deleted. | |||||
| 1045 | sub _install_file :Private { | |||||
| 1046 | 1315 | 6631 | my ($self, $tmp, $outfile) = @_; | |||
| 1047 | ||||||
| 1048 | # The eval must not overwrite the caller's $@ | |||||
| 1049 | 1315 | 924 | local $@; | |||
| 1050 | ||||||
| 1051 | # File::Temp creates files as 0600; give the CSV the permissions a | |||||
| 1052 | # normal open() would have, i.e. 0666 less the umask | |||||
| 1053 | 1315 | 766 | my $ok = eval { | |||
| 1054 | 1315 | 1840 | close $tmp; | |||
| 1055 | 1315 | 33143 | chmod $FILE_MODE & ~umask(), $tmp->filename(); | |||
| 1056 | 1315 | 40752 | rename $tmp->filename(), $outfile; | |||
| 1057 | 1309 | 41431 | 1; | |||
| 1058 | }; | |||||
| 1059 | 1315 | 3829 | $self->_croak_i18n('write_failed', { params => [$outfile, _os_error($@)] }) unless $ok; | |||
| 1060 | ||||||
| 1061 | 1309 | 1377 | $tmp->unlink_on_destroy(0); | |||
| 1062 | 1309 | 3511 | return $self; | |||
| 1063 | 34 34 34 | 5186 20 80 | } | |||
| 1064 | ||||||
| 1065 | # _count_rows | |||||
| 1066 | # Purpose: Ask mdb-count how many rows a table has. | |||||
| 1067 | # Entry Criteria: $self->{programs}{'mdb-count'} is set. | |||||
| 1068 | # Exit Status: Returns a non-negative integer; croaks if mdb-count fails. | |||||
| 1069 | # Side Effects: Runs mdb-count. | |||||
| 1070 | sub _count_rows :Private { | |||||
| 1071 | 57 | 549 | my ($self, $database, $table) = @_; | |||
| 1072 | ||||||
| 1073 | 57 | 81 | my $stdout = ''; | |||
| 1074 | 57 | 200 | $self->_run_program($MDB_COUNT, [$END_OF_OPTIONS, $database, $table], \$stdout); | |||
| 1075 | ||||||
| 1076 | # mdb-count prints just the number (perhaps with spaces around it). | |||||
| 1077 | # Anything else - nothing, "-5", an error text - is not a count, and | |||||
| 1078 | # must not quietly become 0: it is reported (as a warning, by | |||||
| 1079 | # _try_count_rows) | |||||
| 1080 | 50 | 522 | my ($rows) = ($stdout // '') =~ /\A\s*(\d+)\s*\z/; | |||
| 1081 | 50 | 130 | defined($rows) or $self->_croak_i18n('count_unreadable', { params => [$self->_printable(substr($stdout // '', 0, $COUNT_SHOWN))] }); | |||
| 1082 | 42 | 152 | return $rows; | |||
| 1083 | 34 34 34 | 5465 18 89 | } | |||
| 1084 | ||||||
| 1085 | # _try_count_rows | |||||
| 1086 | # Purpose: Count a table's rows, where a failure is only a warning. | |||||
| 1087 | # Row counts are optional extras: they must never make an | |||||
| 1088 | # exported table count as failed, nor end a dry run. | |||||
| 1089 | # Entry Criteria: $self->{programs}{'mdb-count'} is set. | |||||
| 1090 | # Exit Status: Returns the count, or undef if it could not be had. | |||||
| 1091 | # Side Effects: Runs mdb-count; on failure warns and logs "Cannot count | |||||
| 1092 | # the rows of T: E". An interruption is not a failure to | |||||
| 1093 | # count: it is passed on, to stop the run. | |||||
| 1094 | sub _try_count_rows :Private { | |||||
| 1095 | 61 | 439 | my ($self, $database, $table) = @_; | |||
| 1096 | ||||||
| 1097 | 61 | 65 | local $@; | |||
| 1098 | 61 61 | 67 154 | my $rows = eval { $self->_count_rows($database, $table) }; | |||
| 1099 | 61 | 2137 | return $rows if defined $rows; | |||
| 1100 | ||||||
| 1101 | 15 | 21 | die $@ if $self->{interrupted}; | |||
| 1102 | 14 | 10 | my $error = $@; | |||
| 1103 | 14 | 16 | chomp $error; | |||
| 1104 | 14 | 27 | $self->_warn('count_failed', { params => [$table, $error] }); | |||
| 1105 | 14 | 21 | return; | |||
| 1106 | 34 34 34 | 5844 15 75 | } | |||
| 1107 | ||||||
| 1108 | # _run_program | |||||
| 1109 | # Purpose: Run an mdbtools program and check that it succeeded. | |||||
| 1110 | # Shared by every call to mdbtools, so that failures are | |||||
| 1111 | # reported consistently. | |||||
| 1112 | # Entry Criteria: $name is a key of $self->{programs}; $args is an arrayref; | |||||
| 1113 | # $stdout is a scalar ref or a filehandle for run3(). | |||||
| 1114 | # Exit Status: Returns $self; croaks if the program exits non-zero or | |||||
| 1115 | # dies from a signal. | |||||
| 1116 | # Side Effects: Runs a child process; writes to $stdout; sets $?. | |||||
| 1117 | sub _run_program :Private { | |||||
| 1118 | 1721 | 7601 | my ($self, $name, $args, $stdout) = @_; | |||
| 1119 | ||||||
| 1120 | # run3 sets $?, and reads captured output back from a temporary file, | |||||
| 1121 | # which changes the handle $. refers to; keep the caller's values | |||||
| 1122 | 1721 | 1744 | local ($?, $.); | |||
| 1123 | ||||||
| 1124 | # Start the program in a clean environment (as perlsec asks, and as | |||||
| 1125 | # taint mode requires): PATH keeps only absolute directory, and variables | |||||
| 1126 | # that can change how a program is started are removed | |||||
| 1127 | # (File::Spec->path and path_sep, not ":": Windows separates PATH with | |||||
| 1128 | # ";" and its paths contain ":", as in C:\\) | |||||
| 1129 | 1721 31646 33230 | 12325 12626 27439 | local $ENV{PATH} = join($Config{path_sep}, map { _untaint($_) } grep { length && File::Spec->file_name_is_absolute($_) } File::Spec->path()); | |||
| 1130 | 1721 | 3764 | local @ENV{@UNSAFE_ENV}; | |||
| 1131 | 1721 | 12461 | delete @ENV{@UNSAFE_ENV}; | |||
| 1132 | ||||||
| 1133 | # A list (not a string) is passed, so no shell ever sees the file or | |||||
| 1134 | # table name and quoting cannot be abused | |||||
| 1135 | 1721 | 12247 | my $stderr = ''; | |||
| 1136 | 1721 1721 | 1281 2260 | run3([$self->{programs}{$name}, @{$args}], \undef, $stdout, \$stderr); | |||
| 1137 | ||||||
| 1138 | # Distinguish "never started" (-1) and a signal from an ordinary | |||||
| 1139 | # non-zero exit status; $! only means something in the first case | |||||
| 1140 | 1715 | 54876234 | my ($status, $reason) = ($?, "$!"); | |||
| 1141 | 1715 | 1442 | chomp $stderr; | |||
| 1142 | 1715 | 1958 | if($status == -1) { | |||
| 1143 | 6 | 13 | $self->_croak_i18n('program_not_run', { params => [$name, $reason] }); | |||
| 1144 | } | |||||
| 1145 | 1709 | 1845 | if($status & 127) { | |||
| 1146 | # While system() waits, Perl ignores INT and QUIT in this process, | |||||
| 1147 | # so Ctrl-C shows up only as the child dying of SIGINT. Treat | |||||
| 1148 | # that as the user stopping the whole run, not as a bad table. | |||||
| 1149 | 14 | 213 | my $signal = (split ' ', $Config{sig_name})[$status & 127] // ''; | |||
| 1150 | 14 | 75 | if($signal eq 'INT' || $signal eq 'QUIT') { | |||
| 1151 | 3 | 5 | $self->{interrupted} = $signal; | |||
| 1152 | 3 | 16 | $self->_croak_i18n('interrupted', { params => [$signal] }); | |||
| 1153 | } | |||||
| 1154 | 11 | 31 | $self->_croak_i18n('program_signalled', { params => [$name, $status & 127] }); | |||
| 1155 | } | |||||
| 1156 | 1695 | 1482 | if($status) { | |||
| 1157 | 33 | 207 | $self->_croak_i18n('program_failed', { params => [$name, $status >> 8, $stderr] }); | |||
| 1158 | } | |||||
| 1159 | 1662 | 19364 | return $self; | |||
| 1160 | 34 34 34 | 7135 12 92 | } | |||
| 1161 | ||||||
| 1162 | # _csv_filename | |||||
| 1163 | # Purpose: Turn a table name into a safe, unique CSV file name. | |||||
| 1164 | # Entry Criteria: $table is a table name (possibly empty). | |||||
| 1165 | # Exit Status: Returns a file name (no directory) ending in ".csv". | |||||
| 1166 | # Side Effects: Records the name in $self->{used_names}. | |||||
| 1167 | # | |||||
| 1168 | # Names are compared case-insensitively, because Windows and macOS file | |||||
| 1169 | # systems are, so "Orders" and "ORDERS" do not overwrite each other. | |||||
| 1170 | # The loop (rather than a single suffix) guarantees uniqueness even when | |||||
| 1171 | # another table is literally called "Orders_2". | |||||
| 1172 | sub _csv_filename :Protected { | |||||
| 1173 | 17111 | 66815 | my ($self, $table) = @_; | |||
| 1174 | ||||||
| 1175 | 17111 | 7447 | my $name = $table // ''; | |||
| 1176 | ||||||
| 1177 | # macOS only accepts file names that are valid UTF-8 (anything else | |||||
| 1178 | # fails with "Illegal byte sequence"). mdbtools gives UTF-8, so other | |||||
| 1179 | # bytes come from a damaged database: each one becomes "_", on every | |||||
| 1180 | # system, so a table gets the same file name everywhere | |||||
| 1181 | 17111 | 8248 | if(!utf8::is_utf8($name)) { | |||
| 1182 | 17109 263 | 15423 173 | $name = $UTF8_CODEC->encode($UTF8_CODEC->decode($name, sub { '_' })); | |||
| 1183 | } | |||||
| 1184 | ||||||
| 1185 | # Replace characters that are illegal somewhere, then strip leading | |||||
| 1186 | # and trailing whitespace; Windows also silently drops trailing dots | |||||
| 1187 | 17111 | 23741 | $name =~ s/$UNSAFE_CHARS_RE/_/g; | |||
| 1188 | 17111 | 11450 | $name =~ s/$BIDI_CONTROLS_RE/_/g; | |||
| 1189 | ||||||
| 1190 | # C1 controls (U+0080-U+009F; U+009B acts like ESC [ on terminals). | |||||
| 1191 | # In a byte string they must be matched as their UTF-8 form, because a | |||||
| 1192 | # bare [\x80-\x9F] would also hit the continuation bytes of ordinary | |||||
| 1193 | # characters such as the Euro sign (E2 82 AC) | |||||
| 1194 | 17111 | 7845 | if(utf8::is_utf8($name)) { | |||
| 1195 | 2 | 2 | $name =~ s/[\x{80}-\x{9F}]/_/g; | |||
| 1196 | } else { | |||||
| 1197 | 17109 | 6680 | $name =~ s/\xC2[\x80-\x9F]/_/g; | |||
| 1198 | } | |||||
| 1199 | 17111 | 6603 | $name =~ s/\A\s+//; | |||
| 1200 | ||||||
| 1201 | # Shorten over-long names (a hostile or corrupt database can give any | |||||
| 1202 | # length), before trailing spaces and dots are removed, as shortening | |||||
| 1203 | # may leave some at the end | |||||
| 1204 | 17111 | 8118 | $name = $self->_shorten_name($name); | |||
| 1205 | 17111 | 7166 | $name =~ s/[\s.]+\z//; | |||
| 1206 | ||||||
| 1207 | # Avoid hidden files, Windows device names and empty names | |||||
| 1208 | 17111 | 5877 | $name =~ s/\A\./_/; | |||
| 1209 | 17111 | 10861 | $name = "_$name" if $name =~ $RESERVED_NAME_RE; | |||
| 1210 | 17111 | 6755 | $name = $UNNAMED unless length $name; | |||
| 1211 | ||||||
| 1212 | # Find the smallest free suffix (_2, _3, ...). Start from where this | |||||
| 1213 | # name's last search ended rather than from 2: every suffix below that | |||||
| 1214 | # point was taken then and is still taken (names are never released | |||||
| 1215 | # during a run), so the answer is the same, but N tables with the same | |||||
| 1216 | # name cost O(N) in total instead of O(N squared). | |||||
| 1217 | 17111 | 6311 | my $used = $self->{used_names}; | |||
| 1218 | 17111 | 6260 | my $base = lc $name; | |||
| 1219 | 17111 | 6159 | my $file = $name . $CSV_SUFFIX; | |||
| 1220 | 17111 | 8033 | if(exists $used->{lc $file}) { | |||
| 1221 | 11698 | 5971 | my $n = $self->{next_suffix}{$base} // 2; | |||
| 1222 | 11698 | 6314 | $n++ while exists $used->{lc "${name}_$n$CSV_SUFFIX"}; | |||
| 1223 | 11698 | 4007 | $file = "${name}_$n$CSV_SUFFIX"; | |||
| 1224 | 11698 | 4618 | $self->{next_suffix}{$base} = $n + 1; | |||
| 1225 | } | |||||
| 1226 | 17111 | 8465 | $used->{lc $file} = 1; | |||
| 1227 | ||||||
| 1228 | 17111 | 12908 | return $file; | |||
| 1229 | 34 34 34 | 7843 18 100 | } | |||
| 1230 | ||||||
| 1231 | # _shorten_name | |||||
| 1232 | # Purpose: Keep a file name within $MAX_NAME_CHARS characters and | |||||
| 1233 | # $MAX_NAME_BYTES bytes of UTF-8, without cutting a | |||||
| 1234 | # character, or a character from its accents (a grapheme, | |||||
| 1235 | # such as a joined emoji), in half. | |||||
| 1236 | # Entry Criteria: $name is a character string, or a byte string of valid | |||||
| 1237 | # UTF-8 (_csv_filename has replaced any other bytes). | |||||
| 1238 | # Exit Status: Returns $name if it fits, otherwise its longest | |||||
| 1239 | # beginning that fits, in the same form (bytes or | |||||
| 1240 | # characters). | |||||
| 1241 | # Side Effects: None. | |||||
| 1242 | sub _shorten_name :Private { | |||||
| 1243 | 17122 | 29688 | my ($self, $name) = @_; | |||
| 1244 | ||||||
| 1245 | 17122 | 7236 | my $chars = utf8::is_utf8($name); | |||
| 1246 | 17122 | 6107 | my $bytes = $name; | |||
| 1247 | 17122 | 7113 | utf8::encode($bytes) if $chars; | |||
| 1248 | ||||||
| 1249 | 17122 | 5781 | my $text = $bytes; | |||
| 1250 | 17122 | 8748 | utf8::decode($text); | |||
| 1251 | 17122 | 12862 | return $name if length($text) <= $MAX_NAME_CHARS && length($bytes) <= $MAX_NAME_BYTES; | |||
| 1252 | ||||||
| 1253 | # Whole graphemes, as many as fit both limits | |||||
| 1254 | 22 | 33 | my ($kept, $count, $size) = ('', 0, 0); | |||
| 1255 | 22 | 435 | foreach my $grapheme ($text =~ /(\X)/g) { | |||
| 1256 | 1271 | 414 | my $encoded = $grapheme; | |||
| 1257 | 1271 | 599 | utf8::encode($encoded); | |||
| 1258 | 1271 | 879 | last if $count + length($grapheme) > $MAX_NAME_CHARS | |||
| 1259 | || $size + length($encoded) > $MAX_NAME_BYTES; | |||||
| 1260 | 1249 | 422 | $kept .= $grapheme; | |||
| 1261 | 1249 | 398 | $count += length $grapheme; | |||
| 1262 | 1249 | 417 | $size += length $encoded; | |||
| 1263 | } | |||||
| 1264 | 22 | 97 | utf8::encode($kept) unless $chars; | |||
| 1265 | 22 | 24 | return $kept; | |||
| 1266 | 34 34 34 | 6538 10 95 | } | |||
| 1267 | ||||||
| 1268 | # _dry_run | |||||
| 1269 | # Purpose: Print the table -> file mapping without writing anything. | |||||
| 1270 | # Entry Criteria: $tables is the arrayref of selected tables. | |||||
| 1271 | # Exit Status: Returns $self. | |||||
| 1272 | # Side Effects: Prints to STDOUT; runs mdb-count when show_counts is on. | |||||
| 1273 | sub _dry_run :Private { | |||||
| 1274 | 66 | 868 | my ($self, $database, $tables) = @_; | |||
| 1275 | ||||||
| 1276 | # Only show the ROWS column when counts are actually available | |||||
| 1277 | 66 | 121 | my $counts = $self->{show_counts}; | |||
| 1278 | 66 | 225 | my $format = $counts | |||
| 1279 | ? "%-${TABLE_COLUMN_WIDTH}s %${ROWS_COLUMN_WIDTH}s %s\n" | |||||
| 1280 | : "%-${TABLE_COLUMN_WIDTH}s %s\n"; | |||||
| 1281 | 66 146 | 162 3172 | my @header = map { $self->i18n($_) } ($counts ? qw(column_table column_rows column_output) : qw(column_table column_output)); | |||
| 1282 | ||||||
| 1283 | # Underline the title to the length of whatever the translation is | |||||
| 1284 | 66 | 2179 | my $title = $self->i18n('dry_run_title'); | |||
| 1285 | 66 | 2938 | print "\n$title\n", '=' x length($title), "\n\n"; | |||
| 1286 | 66 | 345 | printf $format, @header; | |||
| 1287 | 66 | 236 | print '-' x $RULE_WIDTH, "\n"; | |||
| 1288 | ||||||
| 1289 | 66 66 | 57 103 | foreach my $table (@{$tables}) { | |||
| 1290 | 89 | 123 | my @row = ($table); | |||
| 1291 | # A count that cannot be had is shown as "?" (with a warning), so a | |||||
| 1292 | # dry run still lists every table and succeeds | |||||
| 1293 | 89 | 149 | push @row, $self->_try_count_rows($database, $table) // $UNKNOWN_COUNT if $counts; | |||
| 1294 | # The table name comes from the database, so show it escaped (the | |||||
| 1295 | # file name is already safe; escaping it too costs nothing) | |||||
| 1296 | 89 | 246 | $row[0] = $self->_printable($row[0]); | |||
| 1297 | 89 | 199 | printf $format, @row, $self->_printable($self->_csv_filename($table)); | |||
| 1298 | } | |||||
| 1299 | 66 | 143 | print "\n"; | |||
| 1300 | ||||||
| 1301 | 66 | 99 | return $self; | |||
| 1302 | 34 34 34 | 7034 13 89 | } | |||
| 1303 | ||||||
| 1304 | # _warn | |||||
| 1305 | # Purpose: Warn the user and record the same text in the log. | |||||
| 1306 | # Entry Criteria: $key is a catalog key; $args an optional i18n() hashref. | |||||
| 1307 | # Exit Status: Returns $self. | |||||
| 1308 | # Side Effects: carp()s; logs at warn level. | |||||
| 1309 | sub _warn :Private { | |||||
| 1310 | 110 | 775 | my ($self, $key, $args) = @_; | |||
| 1311 | ||||||
| 1312 | 110 | 303 | $self->_carp_i18n($key, $args); | |||
| 1313 | 110 | 181 | return $self->_log(warn => $key, $args); | |||
| 1314 | 34 34 34 | 5407 13 77 | } | |||
| 1315 | ||||||
| 1316 | # _log | |||||
| 1317 | # Purpose: Send a localised message to the logger, if there is one. | |||||
| 1318 | # Entry Criteria: $level is a logger method name (debug, info or warn). | |||||
| 1319 | # Exit Status: Returns $self. | |||||
| 1320 | # Side Effects: Calls $self->{logger}->$level(). | |||||
| 1321 | sub _log :Private { | |||||
| 1322 | 1648 | 7228 | my ($self, $level, $key, $args) = @_; | |||
| 1323 | ||||||
| 1324 | 1648 | 1732 | my $logger = $self->{logger} or return $self; | |||
| 1325 | ||||||
| 1326 | # Logging is secondary: a logger that dies (full disk, closed socket) | |||||
| 1327 | # must not make a finished export look failed. Say so once and stop | |||||
| 1328 | # using it. | |||||
| 1329 | 174 | 136 | local $@; | |||
| 1330 | # Escaped, so a hostile table name cannot forge log lines (CR, LF) | |||||
| 1331 | # or send escape sequences to whoever views the log | |||||
| 1332 | 174 174 169 | 143 523 13632 | if(!eval { $logger->$level($self->_printable($self->i18n($key, $args))); 1 }) { | |||
| 1333 | 5 | 19 | my $error = $@; | |||
| 1334 | 5 | 6 | chomp $error; | |||
| 1335 | 5 | 4 | delete $self->{logger}; | |||
| 1336 | 5 | 10 | $self->_carp_i18n('log_failed', { params => [$error] }); | |||
| 1337 | } | |||||
| 1338 | 174 | 166 | return $self; | |||
| 1339 | 34 34 34 | 5824 14 84 | } | |||
| 1340 | ||||||
| 1341 | # _validation_reason | |||||
| 1342 | # Purpose: Reduce a Params::Validate::Strict error to its meaning, | |||||
| 1343 | # e.g. "Parameter 'encoding' (latin1) must be one of utf8, | |||||
| 1344 | # utf8-bom, cp1252", dropping the module's own prefix | |||||
| 1345 | # ("Params::Validate::Strict line N: validate_strict: ") | |||||
| 1346 | # and the Perl location it appends. | |||||
| 1347 | # Entry Criteria: $error is the exception from validate_strict. | |||||
| 1348 | # Exit Status: Returns a one-line string. | |||||
| 1349 | # Side Effects: None. A plain function, not a method. | |||||
| 1350 | sub _validation_reason :Private { | |||||
| 1351 | 92 | 28537 | my $error = shift; | |||
| 1352 | ||||||
| 1353 | 92 | 51 | my $reason = "$error"; | |||
| 1354 | 92 | 131 | $reason =~ s/\AParams::Validate::Strict line \d+: //; | |||
| 1355 | 92 | 78 | $reason =~ s/\Avalidate_strict: //; | |||
| 1356 | 92 | 142 | $reason =~ s/ at \S+ line \d+\.?\n?\z//; | |||
| 1357 | 92 | 50 | chomp $reason; | |||
| 1358 | 92 | 117 | return $reason; | |||
| 1359 | 34 34 34 | 5746 14 76 | } | |||
| 1360 | ||||||
| 1361 | # _untaint | |||||
| 1362 | # Purpose: Mark a value that has already been validated as safe for | |||||
| 1363 | # taint mode. | |||||
| 1364 | # Entry Criteria: $value has been checked by the caller (see each call). | |||||
| 1365 | # Exit Status: Returns the untainted value; returns $value unchanged if | |||||
| 1366 | # it is empty or contains a NUL (it will then fail safely | |||||
| 1367 | # at the operating system, still tainted). | |||||
| 1368 | # Side Effects: None. A plain function, not a method. | |||||
| 1369 | sub _untaint :Private { | |||||
| 1370 | 34650 | 66537 | my $value = shift; | |||
| 1371 | ||||||
| 1372 | 34650 | 38432 | return ($value // '') =~ $UNTAINT_RE ? $1 : $value; | |||
| 1373 | 34 34 34 | 4601 16 66 | } | |||
| 1374 | ||||||
| 1375 | # _os_error | |||||
| 1376 | # Purpose: Get a readable OS error from an autodie exception or string. | |||||
| 1377 | # Entry Criteria: $error is whatever eval left in $@. | |||||
| 1378 | # Exit Status: Returns a string. | |||||
| 1379 | # Side Effects: None. A plain function, not a method. | |||||
| 1380 | sub _os_error :Private { | |||||
| 1381 | 14 | 3329 | my $error = shift; | |||
| 1382 | ||||||
| 1383 | # autodie::exception keeps the original $! for us. blessed(), not | |||||
| 1384 | # ref(): asking an unblessed reference ->can() would die and hide | |||||
| 1385 | # the real error. | |||||
| 1386 | 14 | 90 | my $text = (blessed($error) && $error->can('errno')) ? $error->errno() : "$error"; | |||
| 1387 | 14 | 22 | chomp $text; | |||
| 1388 | 14 | 29 | return $text; | |||
| 1389 | 34 34 34 | 5408 13 69 | } | |||
| 1390 | ||||||
| 1391 | 1; | |||||
| 1392 | ||||||