| File: | blib/lib/App/Access2CSV.pm |
| Coverage: | 96.8% |
| line | stmt | bran | cond | sub | time | code |
|---|---|---|---|---|---|---|
| 1 | package App::Access2CSV; | |||||
| 2 | ||||||
| 3 | 40 40 40 | 1239751 21 180 | use strict; | |||
| 4 | 40 40 40 | 32 12 430 | use warnings; | |||
| 5 | 40 40 40 | 1281 47615 41 | use autodie qw(:all); | |||
| 6 | ||||||
| 7 | # Sub::Private must be in enforce mode before it is loaded, so that | |||||
| 8 | # private class methods still work through $class->method dispatch | |||||
| 9 | 40 | 92792 | BEGIN { $Sub::Private::config{mode} = 'enforce' } | |||
| 10 | ||||||
| 11 | # Inherit i18n() and the protected _croak_i18n/_carp_i18n helpers | |||||
| 12 | 40 40 40 | 46 26 55 | use parent 'App::Access2CSV::I18N'; | |||
| 13 | ||||||
| 14 | 34 32 32 | 6411 26 231 | use App::Access2CSV::Exporter; | |||
| 15 | 32 32 32 | 30 17 594 | use Errno qw(EISDIR); | |||
| 16 | 32 32 32 | 20 15 259 | use Fcntl qw(O_APPEND O_CREAT O_WRONLY); | |||
| 17 | 32 32 32 | 23 10 402 | use File::Temp; | |||
| 18 | 32 32 32 | 6736 78549 33 | use Getopt::Long qw(GetOptionsFromArray); | |||
| 19 | 32 32 32 | 1219 11 220 | use IO::Handle; | |||
| 20 | 32 31 31 | 9062 651028 211 | use Log::Abstraction; | |||
| 21 | 31 31 31 | 4599 298535 601 | use Pod::Usage qw(pod2usage); | |||
| 22 | 31 31 31 | 54 6 286 | use Readonly; | |||
| 23 | 31 31 31 | 25 7 216 | use Return::Set qw(set_return); | |||
| 24 | 31 31 31 | 20 11 188 | use Scalar::Util qw(blessed); | |||
| 25 | 31 31 31 | 17 14 339 | use Sub::Private; | |||
| 26 | ||||||
| 27 | our $VERSION = '0.001.1'; | |||||
| 28 | ||||||
| 29 | # --------------------------------------------------------------------- | |||||
| 30 | # Roadmap (from the pre-release gap analysis) | |||||
| 31 | # | |||||
| 32 | # Features | |||||
| 33 | # TODO: Pass mdbtools' CSV options through: --delimiter, --quote, | |||||
| 34 | # --date-format, --no-header, binary column handling (mdb-export's | |||||
| 35 | # -d, -q, -D, -H and -b). | |||||
| 36 | # TODO: --schema: write each table's definition next to its data, using | |||||
| 37 | # mdb-schema. | |||||
| 38 | # TODO: Export one table to standard output (--table X --output -), for | |||||
| 39 | # use in pipes. | |||||
| 40 | # TODO: --gzip, or a single ZIP file holding all the CSV files. | |||||
| 41 | # TODO: More output formats: JSON Lines, and SQL INSERT statements | |||||
| 42 | # (mdb-export -I). | |||||
| 43 | # TODO: --jobs N: export several tables at once. Starting one mdbtools | |||||
| 44 | # process per table is the main cost of a run. | |||||
| 45 | # TODO: A second back end using ODBC (DBD::ODBC with the Microsoft Access | |||||
| 46 | # driver), so Windows does not need mdbtools. | |||||
| 47 | # TODO: Ship at least one real translation (e.g. German) to prove the | |||||
| 48 | # message catalog end to end. | |||||
| 49 | # TODO: Read settings from a configuration file via Object::Configure; | |||||
| 50 | # %DEFAULTS is already laid out for it. | |||||
| 51 | # TODO: Optionally exit 130 on Ctrl-C (the shell convention), not 3. | |||||
| 52 | # | |||||
| 53 | # Technical debt | |||||
| 54 | # TODO: Sub::Private/Sub::Protected hide the real subs from Devel::Cover | |||||
| 55 | # (worked around by the CHECK block in I18N.pm) and refuse | |||||
| 56 | # Test::Mockingbird hooks unless $Sub::*::BYPASS is set. Fix both in | |||||
| 57 | # Sub::Private/Sub::Protected, then drop the workaround. | |||||
| 58 | # TODO: Remove the IO::Handle::flush workaround in t/edge_cases.t once the | |||||
| 59 | # Test::Mockingbird fix for mocking inherited methods is released. | |||||
| 60 | # TODO: IPC::Run3 does not reveal the child's process ID, so after SIGTERM | |||||
| 61 | # the running mdb-export is not stopped. Moving to IPC::Run or | |||||
| 62 | # IPC::Open3 would allow that, and streaming for --jobs. | |||||
| 63 | # TODO: I18N.pm also holds helpers unrelated to messages (_printable, | |||||
| 64 | # _interrupt_signals, the coverage CHECK block); move them to a small | |||||
| 65 | # shared module, and remove the duplicated control-character patterns | |||||
| 66 | # (I18N.pm and Exporter.pm) at the same time. | |||||
| 67 | # TODO: The test files repeat the same helpers (slurp, new_database, cli, | |||||
| 68 | # stand-in set-up); a shared t/lib helper module would shrink them and | |||||
| 69 | # remove most Windows skips in one place. | |||||
| 70 | # TODO: Tests read exporter internals ($e->{show_counts}, {used_names}); | |||||
| 71 | # small read-only accessors would make refactoring safer. | |||||
| 72 | # --------------------------------------------------------------------- | |||||
| 73 | ||||||
| 74 | # Stop Carp from reporting errors against the access-control wrappers | |||||
| 75 | our @CARP_NOT = qw(Sub::Private Sub::Protected App::Access2CSV::I18N); | |||||
| 76 | ||||||
| 77 | # Exit statuses, documented in the POD below | |||||
| 78 | Readonly::Scalar my $EXIT_OK => 0; | |||||
| 79 | Readonly::Scalar my $EXIT_FAILURE => 1; | |||||
| 80 | Readonly::Scalar my $EXIT_USAGE => 2; | |||||
| 81 | Readonly::Scalar my $EXIT_FATAL => 3; | |||||
| 82 | ||||||
| 83 | # Pod::Usage verbosity levels for --help and --man, and for usage errors | |||||
| 84 | Readonly::Scalar my $POD_SYNOPSIS => 0; | |||||
| 85 | Readonly::Scalar my $POD_OPTIONS => 1; | |||||
| 86 | Readonly::Scalar my $POD_FULL => 2; | |||||
| 87 | ||||||
| 88 | # The database name that means "read it from standard input" | |||||
| 89 | Readonly::Scalar my $STDIN_NAME => '-'; | |||||
| 90 | ||||||
| 91 | # Standard input is copied, in chunks of this many bytes, to a private | |||||
| 92 | # temporary file named from this template (mdbtools need a real file) | |||||
| 93 | Readonly::Scalar my $READ_CHUNK => 65_536; | |||||
| 94 | Readonly::Scalar my $STDIN_TEMPLATE => 'access2csv-stdin-XXXXXX'; | |||||
| 95 | ||||||
| 96 | # Log levels: --verbose adds the debug messages | |||||
| 97 | Readonly::Scalar my $LOG_LEVEL => 'info'; | |||||
| 98 | Readonly::Scalar my $LOG_LEVEL_VERBOSE => 'debug'; | |||||
| 99 | ||||||
| 100 | # Command-line defaults; the flat layout is compatible with Object::Configure | |||||
| 101 | Readonly::Hash my %DEFAULTS => ( | |||||
| 102 | output_dir => '.', | |||||
| 103 | overwrite => 0, | |||||
| 104 | verbose => 0, | |||||
| 105 | dry_run => 0, | |||||
| 106 | show_counts => 0, | |||||
| 107 | progress => 1, | |||||
| 108 | encoding => 'utf8', | |||||
| 109 | log => 'access2csv.log', | |||||
| 110 | ); | |||||
| 111 | ||||||
| 112 | =encoding utf8 | |||||
| 113 | ||||||
| 114 - 620 | =head1 NAME
App::Access2CSV - Export the tables of a Microsoft Access database to CSV files
=head1 VERSION
Version 0.001.1
=head1 SYNOPSIS
# Export every table to the current directory
access2csv shop.accdb
# See what would be written, with row counts, without writing anything
access2csv --dry-run --show-counts shop.accdb
# Export only two tables, into a directory called "exports"
access2csv --output-dir exports --table Customers --table Orders shop.mdb
# Make files that Excel opens correctly, replace old files, no log file
access2csv --encoding utf8-bom --overwrite --no-log shop.accdb
# Read the database from standard input ("-"), e.g. from a download
curl -s https://example.com/shop.accdb | access2csv --output-dir exports -
# Nightly job: quiet, with a log in a fixed place, and stop on failure
access2csv --no-progress --log /var/log/access2csv.log \
--output-dir /srv/exports --overwrite shop.accdb || exit 1
=head1 DESCRIPTION
Microsoft Access keeps its data in C<.mdb> or C<.accdb> files.
C<access2csv> reads one of these files and writes one CSV file
(comma-separated values, a plain-text table) for each table in it.
It does not read the Access file itself. It runs three small programs
from the free B<mdbtools> package:
=over 4
=item * C<mdb-tables> - to get the list of tables
=item * C<mdb-export> - to get the data of each table as CSV
=item * C<mdb-count> - to count rows (only when you use B<--show-counts>)
=back
These programs must be installed and must be in your C<PATH>.
Access also keeps its own internal tables in the file. Their names start
with C<MSys>, C<USys> or C<~>. They are skipped.
=head2 How the CSV files are named
Each file has the name of its table plus C<.csv>, for example
C<Orders.csv>. Some characters are not allowed in file names on some
computers (C<< < > : " / \ | ? * >> and control characters). They are
changed to C<_>, and so are invisible text-direction controls (such as
U+202E, "right-to-left override"), which could make a file name look
like something else. Bytes that are not valid UTF-8 (only a damaged
database has them, and macOS cannot store them in a file name) are also
changed to C<_>. Spaces and dots at the end, and spaces at the start,
are removed. A name such as C<CON> or C<NUL> (reserved on Windows) gets a
C<_> in front. An empty name becomes C<unnamed>.
Access names are at most 64 characters, so a longer name can only come
from a damaged database. It is shortened to 64 characters (and to 240
bytes, to fit the file name limit of most systems), without cutting a
character in half.
If two tables would get the same file name, the second one gets C<_2>,
the third C<_3>, and so on. Upper and lower case count as the same here,
because Windows and macOS treat C<Orders.csv> and C<ORDERS.csv> as one file.
=head2 Reading the database from standard input
If the database name is C<->, the database is read from standard input
instead of a file, so it can be piped in. mdbtools can only read a real
file, so the data is first copied to a private temporary file (readable
by you only) in the temporary directory (C<TMPDIR>, or F</tmp>), and that
copy is deleted when the program ends - whether it succeeds, fails or is
interrupted.
C<-> is refused if standard input is a terminal (there is nothing to
read but the keyboard), and a directory or empty input is an error. To use
a file that is really called C<->, write F<./->.
=head2 How files are written
Each file is first written to a hidden temporary file (its name starts
with C<.access2csv->) in the output directory. Only when it is complete
is it renamed to its real name. So if something goes wrong, you never
get a half-written CSV file, and an old file is only replaced by a
complete new one.
=head1 REQUIREMENTS
The mdbtools programs C<mdb-tables> and C<mdb-export> must be installed
and in your C<PATH>; C<mdb-count> is needed only for B<--show-counts>.
They are not Perl modules, so the CPAN installer cannot install them for
you. Install them with your system's package manager, for example:
sudo apt install mdbtools # Debian, Ubuntu
sudo dnf install mdbtools # Fedora
brew install mdbtools # macOS (Homebrew)
pacman -S mingw-w64-x86_64-mdbtools # Windows (MSYS2)
Without them the program stops with "Required program not found in
PATH". Project home: L<https://github.com/mdbtools/mdbtools>.
=head1 USING FROM PERL
The program is a very thin wrapper. You can call the same code from Perl:
use App::Access2CSV;
my $status = App::Access2CSV->run('--no-log', '--output-dir', 'out', 'shop.accdb');
For more control, use L<App::Access2CSV::Exporter> directly.
=head1 OPTIONS
=over 4
=item B<--output-dir> I<DIR>
The directory to write the CSV files to. It is created if it does not exist.
Default: the directory.
=item B<--table> I<NAME>
Export only this table. You can use this option more than once.
Names must match exactly, including upper and lower case. A name that is
not in the database gives a warning.
=item B<--overwrite>
Replace CSV files that already exist. Without this option, a table whose
CSV file already exists is not exported, and it counts as a failure.
=item B<--verbose>
Write more detail to the log (where each mdbtools program was found).
Also show the Perl file and line number in fatal error messages.
=item B<--dry-run>
Only print a list of the tables and the file names they would get.
Nothing is written. The output directory is not created.
=item B<--show-counts>
Show the number of rows of each table: in the dry-run list, and in the
log. This needs C<mdb-count>. Without it you get a warning, and the
export goes on without counts.
=item B<--no-progress>
Do not print the C<[1/5] Customers> progress lines. (These lines go to
standard error, not standard output.)
=item B<--encoding> I<utf8|utf8-bom|cp1252>
The character encoding of the CSV files. See L</ENCODING>.
Default: C<utf8>.
=item B<--log> I<FILE>
Add log messages to the end of I<FILE>. Default: F<access2csv.log> in
the directory. An empty name (C<--log ''>) means no log. I<FILE>
must not be a symbolic link (see L</SECURITY>).
=item B<--no-log>
Do not write a log file.
=item B<--help>, B<-h>
Print the synopsis and the options, then stop.
=item B<--man>
Print this whole manual, then stop.
=item B<--version>
Print the version ("access2csv version 0.001.1"), then stop.
=back
=head1 EXIT STATUS
The program ends with one of these numbers. Scripts can test it.
0 Every selected table was exported. Also used for --dry-run,
--help, --man and --version.
1 At least one table was not exported. The other tables were.
2 The command line was wrong, for example an unknown option, an
invalid value (--encoding latin1), no database name, or "-" while
standard input is a terminal. Nothing has been done.
3 A fatal error happened before any table was exported, for example
the database does not exist or mdbtools is not installed.
=head1 ENCODING
=head2 The data in the CSV files
mdbtools gives the table data as UTF-8, the encoding that can hold every
character, including accented letters, Chinese and Japanese text, and
emoji.
=over 4
=item * C<utf8> (the default) - the data is copied exactly as mdbtools
gives it. Every character, including emoji, is kept.
=item * C<utf8-bom> - the same, plus three bytes at the very start of each
file (a "byte order mark"). These bytes tell Microsoft Excel that the file
is UTF-8. Without them, Excel may show accented letters wrongly. Some
other programs show the mark as strange characters in the first column
name.
=item * C<cp1252> - Windows-1252, an old Western European encoding. It has
only 256 characters: English letters, most Western European accented
letters, and a few symbols such as the Euro sign. It has no Greek,
Cyrillic, Chinese, Japanese or emoji. If a table contains a character
that Windows-1252 cannot hold, that table is B<not> exported, and the
error message gives the line number. Nothing is silently replaced.
=back
=head2 Names on the command line
Database paths, directory names, log file names and table names are used
exactly as the operating system gives them to the program (as bytes).
On Linux and macOS, where the terminal uses UTF-8, names with accented
letters, non-Latin scripts and emoji work. On Windows, the command line
uses the system code page, so names outside that code page may not work.
=head2 Messages
All messages that the program prints and logs are in plain ASCII English.
=head1 ENVIRONMENT
=over 4
=item C<PATH>
Used to find C<mdb-tables>, C<mdb-export> and C<mdb-count>. Only
absolute directories in C<PATH> are used: relative entries such as C<.> are
ignored, so a program planted in the directory is never run.
=item C<LANGUAGE>, C<LC_ALL>, C<LC_MESSAGES>, C<LANG>
Choose the language of messages (see L<App::Access2CSV::I18N>). Only the
language code at the start is used; any other value means English.
=item C<TMPDIR>
Where a database read from standard input (C<->) is copied while it is
exported. Default: F</tmp>.
=item C<MDB_ICONV>
Not read by this program, but by mdbtools: it sets the character set
mdbtools converts to. Leave it unset, so that the output is UTF-8.
=back
=head1 SECURITY
The program treats the database as untrusted: an Access file received
from someone else may contain table names and data designed to cause
harm.
=over 4
=item * B<No shell, no option injection.> Programs are run directly
(never through a shell), and table and file names are passed after a
C<--> marker, so names containing C<; | $( ) `> or starting with C<->
are only ever names.
=item * B<No planted programs.> Relative C<PATH> entries are ignored
(see L</ENVIRONMENT>). The mdbtools programs are started with a cleaned
environment: C<PATH> holds only absolute directories, and C<IFS>, C<CDPATH>,
C<ENV> and C<BASH_ENV> are removed.
=item * B<Taint mode.> The program runs under Perl's taint mode
(C<perl -T>). Every outside value - the database path, table names,
C<--output-dir>, C<--log> and the program paths found in C<PATH> - is
checked first and only then marked as safe. Under C<-T>, Perl also
refuses to start mdbtools while C<PATH> contains a directory other users can
write to; the program then stops with "Insecure directory in
$ENV{PATH}".
=item * B<Private copies of piped input.> A database read from standard
input is copied with L<File::Temp> (a new, unpredictable name, readable
by you only) and deleted when the program ends, also after an error or
an interruption.
=item * B<Safe file names.> Table names cannot place a file outside the
output directory, and control characters - including invisible
text-direction controls and C1 controls - are replaced by C<_>.
=item * B<Safe terminal and log output.> Table names and mdbtools error
text are printed with control characters shown as escapes such as
C<\x1B>. So a table name cannot retitle or clear your terminal, hide
text, or forge lines in the log.
=item * B<No writing through symbolic links.> If the log file is a
symbolic link (for example one planted in a shared directory such as
F</tmp>), the program stops instead of writing to the file it points at.
Existing CSV files that are links are replaced, never written through.
=item * B<Spreadsheet formulas are NOT neutralised.> A value such as
C<=cmd|' /C calc'!A0> is copied into the CSV exactly as it is in the
database, because changing data would corrupt genuine values. Some
spreadsheet programs run such formulas when a CSV is opened. Do not open
CSV files exported from an untrusted database in a spreadsheet without
checking them, or import them as text.
=back
=head1 COMMON PITFALLS
=over 4
=item * B<A log file appears in the directory directory.> By default the log is
F<access2csv.log> in the directory you run the program from. Use B<--log> to
choose another place, or B<--no-log>.
=item * B<The second run fails.> If the CSV files already exist, each table
fails (exit status 1) unless you give B<--overwrite>.
=item * B<--table does not find my table.> Table names are case-sensitive:
C<--table orders> does not match C<Orders>. Use B<--dry-run> to see the
exact names.
=item * B<A file is called Orders_2.csv.> Two tables had names that give
the same file name (for example C<Orders> and C<ORDERS>, or C<A/B> and
C<A:B>).
=item * B<Progress lines appear even though I redirected the output.>
Progress lines go to standard error. Use B<--no-progress>, or redirect
standard error too (C<2E<gt>/dev/null>).
=item * B<run() does not end the program.> When calling from Perl,
C<< App::Access2CSV->run(...) >> returns the exit status. It does not
call C<exit>. Write C<< exit App::Access2CSV->run(@ARGV) >> if you want
the program to end.
=item * B<Pass a list, not an array reference.> Write
C<< App::Access2CSV->run(@args) >>, not C<< App::Access2CSV->run(\@args) >>.
=item * B<A file called "-".> C<-> means standard input, even after
C<-->. Write F<./-> for a file with that name.
=item * B<Piped databases need temporary space.> The whole database is
copied to the temporary directory first; if that directory is small, set
C<TMPDIR> to one with room.
=back
=head1 METHODS
=head2 run
=head3 Purpose
This is the whole C<access2csv> program. It reads the command-line
options, opens the log, and runs an L<App::Access2CSV::Exporter>.
=head3 Arguments
The command-line arguments, as a list of strings (normally C<@ARGV>).
Your array is copied first, so it is not changed.
=head3 Returns
A number from 0 to 3, as described in L</EXIT STATUS>.
C<run> never calls C<exit> itself.
=head3 Side Effects
=over 4
=item * Everything that L<App::Access2CSV::Exporter/run> does: it creates
the output directory, writes CSV files, and prints progress to standard error.
=item * It prints help or usage text (help to standard output, usage errors
to standard error).
=item * It prints a fatal error, if there is one, to standard error as one
line that starts with C<access2csv:>.
=item * It creates or adds to the log file, unless logging is off.
=item * It leaves your C<$@>, C<$!>, C<$_> and any pending C<alarm> as
they were.
=back
=head3 Usage
exit App::Access2CSV->run(@ARGV);
=head3 EXAMPLE
use App::Access2CSV;
# Export to ./out without a log file, then check what happened
my $status = App::Access2CSV->run('--output-dir', 'out', '--no-log', 'shop.accdb');
if($status == 0) {
print "All tables were exported\n";
} elsif($status == 1) {
print "Some tables could not be exported\n";
} elsif($status == 2) {
print "The arguments were wrong\n";
} else {
print "Nothing was exported\n";
}
=head3 API SPECIFICATION
=head4 Input
{
argv => {
type => 'arrayref',
optional => 1,
element_type => 'string',
description => 'Command-line arguments, passed as a list',
},
}
Valid and invalid values (tested in F<t/domain.t>):
database names exactly 1; 0 or 2 or more give exit status 2.
"-" means standard input (exit 2 if it is a
terminal, 3 if it is empty or unreadable)
--encoding utf8, utf8-bom or cp1252; anything else gives exit 2
--table 0 times (all tables), once, or many times; names
may be non-ASCII
--log a file name; '' means no log, like --no-log
=head4 Output
{
type => 'integer',
min => 0,
max => 3,
}
=head3 MESSAGES
+-------------------------------------+-------------------------------+------------------------------+
| Message | Meaning | What to do |
+-------------------------------------+-------------------------------+------------------------------+
| Unknown option: X (exit 2) | X is not an option of this | See --help |
| | program | |
| Option X requires an argument | An option such as --log was | Give a value after it |
| (exit 2) | the last word | |
| Invalid setting: REASON (exit 2) | An option value is not | Use a documented value (see |
| | allowed, e.g. --encoding | OPTIONS) |
| | latin1; REASON says which | |
| Missing database filename (exit 2) | No database name was given, | Give exactly one database |
| | it was empty, or more than | |
| | one was given | |
| Standard input is a terminal: pipe | "-" was given, but nothing is | Pipe the database in, or |
| the database in, or give its file | piped in | give its file name |
| name (exit 2) | | |
| access2csv: Standard input is empty:| "-" was given, but the pipe | Check the command that |
| no database was piped in (exit 3) | delivered nothing | produces the database |
| access2csv: Cannot read standard | Reading the pipe failed; E is | See E |
| input: E (exit 3) | the reason ("Is a directory" | |
| | if a directory was given) | |
| access2csv: Interrupted by SIGx | Stopped (Ctrl-C, kill) while | Run again |
| while reading the database from | waiting for piped input; the | |
| standard input (exit 3) | partial copy was deleted | |
| access2csv: Cannot open log file F: | The log file cannot be | Use --log with another file, |
| E (exit 3) | written; E is the reason from | or --no-log |
| | the operating system, "no | |
| | logger was created", or "it | |
| | is a symbolic link" | |
| access2csv: MESSAGE (exit 3) | Any fatal error from the | See MESSAGES in |
| | exporter | App::Access2CSV::Exporter |
+-------------------------------------+-------------------------------+------------------------------+
=head3 PSEUDOCODE
options := default settings
read the command line into options
if the command line is wrong: print usage, return 2
if --help or --man: print the documentation, return 0
if there is not exactly one database name: print usage, return 2
try:
open the log, unless logging is off
status := new Exporter(options).run(database)
if that failed:
print "access2csv: <reason>" to standard error
status := 3
return status
=cut | |||||
| 621 | ||||||
| 622 | sub run { | |||||
| 623 | 198 | 7618090 | my ($class, @argv) = @_; | |||
| 624 | ||||||
| 625 | # Option parsing, file tests and the eval below would otherwise leave | |||||
| 626 | # their marks in the caller's $@ and $! | |||||
| 627 | 198 | 1216 | local ($@, $!); | |||
| 628 | ||||||
| 629 | # Parsing may already decide the outcome (--help, bad options, ...) | |||||
| 630 | 198 | 368 | my %opt = %DEFAULTS; | |||
| 631 | 198 | 4308 | my $status = $class->_parse_options(\@argv, \%opt); | |||
| 632 | ||||||
| 633 | # CHECKING SETTINGS: a bad option value (e.g. --encoding latin1) is a | |||||
| 634 | # command-line mistake like any other: a usage error (exit 2), found | |||||
| 635 | # before anything with a side effect happens - before standard input | |||||
| 636 | # is copied and before the log file is created | |||||
| 637 | 198 1576 1774 | 271 860 710 | my %settings = map { $_ => $opt{$_} } grep { $_ ne 'log' } keys %opt; | |||
| 638 | 198 150 144 | 335 433 5877 | if(!defined($status) && !eval { App::Access2CSV::Exporter->new(%settings); 1 }) { | |||
| 639 | 6 | 538 | $status = $class->_usage($EXIT_USAGE, $POD_SYNOPSIS, _strip_location($@)); | |||
| 640 | } | |||||
| 641 | ||||||
| 642 | 198 | 196 | if(!defined $status) { | |||
| 643 | # Any croak from here on is a fatal error: report it, don't die | |||||
| 644 | 144 | 119 | $status = eval { | |||
| 645 | # READING STDIN: "-" means standard input. mdbtools can only | |||||
| 646 | # read a real file, so the data is copied to a private temporary | |||||
| 647 | # file first; the copy is deleted when $piped goes out of scope, | |||||
| 648 | # whatever happens | |||||
| 649 | 144 | 158 | my $piped = ($argv[0] eq $STDIN_NAME) ? $class->_read_stdin() : undef; | |||
| 650 | 132 | 152 | my $database = $piped ? $piped->filename() : $argv[0]; | |||
| 651 | ||||||
| 652 | # OPENING LOG, then EXPORTING | |||||
| 653 | 132 | 289 | my $logger = $class->_make_logger(\%opt); | |||
| 654 | 118 | 215 | App::Access2CSV::Exporter->new(%settings, ($logger ? (logger => $logger) : ()))->run($database); | |||
| 655 | }; | |||||
| 656 | 144 | 20351 | $status = $class->_report_fatal($@, $opt{verbose}) unless defined $status; | |||
| 657 | } | |||||
| 658 | ||||||
| 659 | 198 | 1258 | return set_return($status, { type => 'integer', min => $EXIT_OK, max => $EXIT_FATAL }); | |||
| 660 | } | |||||
| 661 | ||||||
| 662 | # _parse_options | |||||
| 663 | # Purpose: Turn the command line into settings. | |||||
| 664 | # Entry Criteria: $argv is an arrayref (modified in place: options are | |||||
| 665 | # removed, leaving the positional arguments); $opt is a | |||||
| 666 | # hashref of defaults. | |||||
| 667 | # Exit Status: Returns undef if the export should go ahead, otherwise | |||||
| 668 | # the exit status to return straight away. | |||||
| 669 | # Side Effects: Fills in $opt; prints help, the manual or usage text. | |||||
| 670 | sub _parse_options :Private { | |||||
| 671 | 208 | 7836 | my ($class, $argv, $opt) = @_; | |||
| 672 | ||||||
| 673 | 208 | 155 | my ($help, $show_version) = (0, 0); | |||
| 674 | my $parsed = GetOptionsFromArray( | |||||
| 675 | $argv, | |||||
| 676 | 'output-dir=s' => \$opt->{output_dir}, | |||||
| 677 | 'table=s@' => \$opt->{tables}, | |||||
| 678 | 'overwrite!' => \$opt->{overwrite}, | |||||
| 679 | 'verbose!' => \$opt->{verbose}, | |||||
| 680 | 'dry-run!' => \$opt->{dry_run}, | |||||
| 681 | 'show-counts!' => \$opt->{show_counts}, | |||||
| 682 | 'progress!' => \$opt->{progress}, | |||||
| 683 | 'encoding=s' => \$opt->{encoding}, | |||||
| 684 | 'log=s' => \$opt->{log}, | |||||
| 685 | 112 | 34468 | 'no-log' => sub { $opt->{log} = undef }, | |||
| 686 | 17 | 4501 | 'help|h' => sub { $help = $POD_OPTIONS }, | |||
| 687 | 5 | 1150 | 'man' => sub { $help = $POD_FULL }, | |||
| 688 | 208 | 1371 | 'version' => \$show_version, | |||
| 689 | ); | |||||
| 690 | ||||||
| 691 | # Only one of these applies; the first match decides the exit status. | |||||
| 692 | # Getopt::Long has already warned about any unknown option. | |||||
| 693 | 208 | 49385 | return $class->_usage($EXIT_USAGE, $POD_SYNOPSIS) unless $parsed; | |||
| 694 | 190 | 223 | return $class->_usage($EXIT_OK, $help) if $help; | |||
| 695 | 171 | 159 | if($show_version) { | |||
| 696 | 1 | 3 | print $class->i18n('version', { params => [$VERSION] }), "\n"; | |||
| 697 | 1 | 46 | return $EXIT_OK; | |||
| 698 | } | |||||
| 699 | # An empty or undefined name is as good as no name at all. length() | |||||
| 700 | # of an empty string is 0, so one length test covers "", and "// ''" | |||||
| 701 | # turns undef into "" first. | |||||
| 702 | 170 170 | 111 584 | if(@{$argv} != 1 || !length($argv->[0] // '')) { | |||
| 703 | 23 | 77 | return $class->_usage($EXIT_USAGE, $POD_SYNOPSIS, $class->i18n('missing_database')); | |||
| 704 | } | |||||
| 705 | ||||||
| 706 | # Reading a database from a terminal would only wait for keystrokes | |||||
| 707 | 147 | 295 | if($argv->[0] eq $STDIN_NAME && -t STDIN) { | |||
| 708 | 0 | 0 | return $class->_usage($EXIT_USAGE, $POD_SYNOPSIS, $class->i18n('stdin_is_terminal')); | |||
| 709 | } | |||||
| 710 | 147 | 172 | return; | |||
| 711 | 31 31 31 | 8124 11 237 | } | |||
| 712 | ||||||
| 713 | # _usage | |||||
| 714 | # Purpose: Print documentation from this module's POD. | |||||
| 715 | # Entry Criteria: $status is the exit status to return; $verbose is a | |||||
| 716 | # Pod::Usage verbosity; $message is an optional error. | |||||
| 717 | # Exit Status: Returns $status. | |||||
| 718 | # Side Effects: Prints to STDOUT (help) or STDERR (errors). | |||||
| 719 | sub _usage :Private { | |||||
| 720 | 57 | 3369 | my ($class, $status, $verbose, $message) = @_; | |||
| 721 | ||||||
| 722 | # The POD lives here, not in bin/access2csv, so point Pod::Usage at | |||||
| 723 | # this file; NOEXIT keeps run() testable | |||||
| 724 | 57 | 207 | pod2usage( | |||
| 725 | -input => __FILE__, | |||||
| 726 | -verbose => $verbose, | |||||
| 727 | -exitval => 'NOEXIT', | |||||
| 728 | -output => $status == $EXIT_OK ? \*STDOUT : \*STDERR, | |||||
| 729 | (defined($message) ? (-message => $message) : ()), | |||||
| 730 | ); | |||||
| 731 | 57 | 1826110 | return $status; | |||
| 732 | 31 31 31 | 5178 17 112 | } | |||
| 733 | ||||||
| 734 | # _make_logger | |||||
| 735 | # Purpose: Create the log, unless logging is switched off. | |||||
| 736 | # Entry Criteria: $opt->{log} is a file name, or undef/'' for no log. | |||||
| 737 | # Exit Status: Returns a Log::Abstraction object or undef; croaks if the | |||||
| 738 | # file cannot be appended to. | |||||
| 739 | # Side Effects: Creates the log file if it does not exist. | |||||
| 740 | sub _make_logger :Private { | |||||
| 741 | 140 | 4559 | my ($class, $opt) = @_; | |||
| 742 | ||||||
| 743 | 140 | 307 | return unless length($opt->{log} // ''); | |||
| 744 | ||||||
| 745 | 45 | 35 | my $file = $opt->{log}; | |||
| 746 | ||||||
| 747 | # Never write through a symbolic link: in a shared directory such as /tmp | |||||
| 748 | # anyone could plant "access2csv.log" pointing at a file of yours | |||||
| 749 | 45 | 257 | if(-l $file) { | |||
| 750 | 4 | 10 | $class->_croak_i18n('log_open_failed', { params => [$file, $class->i18n('log_is_symlink')] }); | |||
| 751 | } | |||||
| 752 | ||||||
| 753 | # The log file is the invoking user's own choice and is not a link, so | |||||
| 754 | # it is untainted (for taint mode); a NUL byte cannot name a file at all | |||||
| 755 | 41 | 130 | ($file) = $file =~ /\A([^\x00]+)\z/s or $class->_croak_i18n('log_open_failed', { params => [$opt->{log}, $class->i18n('invalid_name')] }); | |||
| 756 | ||||||
| 757 | # Open the log once, here, and give Log::Abstraction the handle, not the | |||||
| 758 | # name. Given a name, it reopens the file for every message (following | |||||
| 759 | # any link planted since the -l test above), rejects names containing | |||||
| 760 | # "..", "$", "!", ";" and the like only when the first message is | |||||
| 761 | # written, and ignores a file it cannot open - losing the log without | |||||
| 762 | # telling anyone. O_NOFOLLOW (where the OS has it) closes the gap | |||||
| 763 | # between the -l test and this open. The eval must not overwrite the | |||||
| 764 | # caller's $@. | |||||
| 765 | 41 | 33 | local $@; | |||
| 766 | 41 | 23 | my $fh; | |||
| 767 | 41 | 27 | eval { | |||
| 768 | 41 | 72 | sysopen $fh, $file, O_WRONLY | O_APPEND | O_CREAT | _no_follow(); | |||
| 769 | 31 | 4834 | 1; | |||
| 770 | } or $class->_croak_i18n('log_open_failed', { params => [$file, _failure_reason($@)] }); | |||||
| 771 | ||||||
| 772 | # Write each line as it is logged, as reopening the file did, so a | |||||
| 773 | # killed run still leaves its log behind | |||||
| 774 | 31 | 192 | $fh->autoflush(1); | |||
| 775 | ||||||
| 776 | # The handle is closed when the last reference to the logger goes | |||||
| 777 | my $logger = Log::Abstraction->new( | |||||
| 778 | logger => { fd => $fh }, | |||||
| 779 | 31 | 1057 | level => $opt->{verbose} ? $LOG_LEVEL_VERBOSE : $LOG_LEVEL, | |||
| 780 | ); | |||||
| 781 | ||||||
| 782 | # The user asked for a log; carrying on without one would silently | |||||
| 783 | # break that promise | |||||
| 784 | 31 | 3667 | $logger or $class->_croak_i18n('log_open_failed', { params => [$file, $class->i18n('logger_unavailable')] }); | |||
| 785 | 27 | 30 | return $logger; | |||
| 786 | 31 31 31 | 6924 16 111 | } | |||
| 787 | ||||||
| 788 | # _read_stdin | |||||
| 789 | # Purpose: Copy the database piped in on standard input to a | |||||
| 790 | # private temporary file, because mdbtools can only read | |||||
| 791 | # a real, seekable file. | |||||
| 792 | # Entry Criteria: STDIN is not a terminal (checked by _parse_options). | |||||
| 793 | # Exit Status: Returns the File::Temp object; the file is deleted when | |||||
| 794 | # the object is destroyed. Croaks if the input is a | |||||
| 795 | # directory or empty, cannot be read, cannot be stored, or | |||||
| 796 | # the copy is interrupted. | |||||
| 797 | # Side Effects: Reads all of STDIN; writes a file (mode 0600) in the | |||||
| 798 | # temporary directory (TMPDIR). | |||||
| 799 | sub _read_stdin :Private { | |||||
| 800 | 17 | 76 | my $class = shift; | |||
| 801 | ||||||
| 802 | # Perl's default action for INT/TERM/... exits at once, which would | |||||
| 803 | # leave the copy behind in the temporary directory. Raise an exception | |||||
| 804 | # instead, so the File::Temp object is destroyed and deletes the file. | |||||
| 805 | 17 | 8 | my $interrupted; | |||
| 806 | 17 17 | 12 35 | my @ours = @{ $class->_interrupt_signals() }; | |||
| 807 | 17 3 3 | 238 566180 30 | local @SIG{@ours} = (sub { $interrupted = $_[0]; die "\n" }) x @ours; | |||
| 808 | ||||||
| 809 | # A directory as standard input ("access2csv - < dir"). On Linux reading | |||||
| 810 | # it fails with EISDIR, but NetBSD (and FreeBSD before 13) return the | |||||
| 811 | # directory's raw entries, which would be copied as if they were a | |||||
| 812 | # database. Refuse it on every system, with the reason Linux gives. | |||||
| 813 | # https://www.cpantesters.org/cpan/report/e9759fca-bda8-11f1-b0a1-a794297aa253 | |||||
| 814 | 17 | 40 | if(-d STDIN) { | |||
| 815 | 4 4 4 | 7 14 23 | $class->_croak_i18n('stdin_read_failed', { params => [do { local $! = EISDIR; "$!" }] }); | |||
| 816 | } | |||||
| 817 | ||||||
| 818 | 13 | 25 | my $copy = File::Temp->new(TEMPLATE => $STDIN_TEMPLATE, TMPDIR => 1, UNLINK => 1); | |||
| 819 | 13 | 1895 | binmode $copy, ':raw'; | |||
| 820 | 13 | 3182 | binmode STDIN, ':raw'; | |||
| 821 | ||||||
| 822 | # Copy in chunks: a large database is never held in memory at once | |||||
| 823 | 13 | 133 | my $total = 0; | |||
| 824 | 13 | 6 | local $@; | |||
| 825 | eval { | |||||
| 826 | 13 | 32 | while(my $got = read(STDIN, my $buffer, $READ_CHUNK)) { | |||
| 827 | 38 38 | 1523 1010 | print {$copy} $buffer; | |||
| 828 | 38 | 17 | $total += $got; | |||
| 829 | } | |||||
| 830 | 10 | 888 | 1; | |||
| 831 | 13 | 11 | } or do { | |||
| 832 | 3 | 33 | $class->_croak_i18n('interrupted_reading', { params => [$interrupted] }) if $interrupted; | |||
| 833 | 0 | 0 | $class->_croak_i18n('stdin_read_failed', { params => [_failure_reason($@)] }); | |||
| 834 | }; | |||||
| 835 | ||||||
| 836 | 10 | 18 | $class->_croak_i18n('stdin_empty') unless $total; | |||
| 837 | ||||||
| 838 | # Make sure every byte reached the file (e.g. the disk may be full) | |||||
| 839 | 5 | 73 | $copy->flush() or $class->_croak_i18n('write_failed', { params => [$copy->filename(), "$!"] }); | |||
| 840 | 5 | 50 | return $copy; | |||
| 841 | 31 31 31 | 6665 19 105 | } | |||
| 842 | ||||||
| 843 | # _strip_location | |||||
| 844 | # Purpose: Remove the " at FILE line N." that Carp appends, which is | |||||
| 845 | # noise for a command-line user. | |||||
| 846 | # Entry Criteria: $text is an error message. | |||||
| 847 | # Exit Status: Returns the message without the location or newline. | |||||
| 848 | # Side Effects: None. A plain function, not a method. | |||||
| 849 | # | |||||
| 850 | # The file name may contain spaces ("My Documents"), so it cannot be | |||||
| 851 | # matched as \S+. Instead: " at ", then the shortest run of characters | |||||
| 852 | # that does not contain another " at ", then " line N." at the very end. | |||||
| 853 | # The (?! at ) guard keeps this linear: each attempt stops at the next | |||||
| 854 | # " at ", so no character is scanned by more than one attempt. | |||||
| 855 | sub _strip_location :Private { | |||||
| 856 | 83 | 240 | my $text = shift; | |||
| 857 | ||||||
| 858 | 83 | 268 | $text =~ s/ | |||
| 859 | [ ] at [ ] # Carp's separator | |||||
| 860 | (?: (?! [ ] at [ ] ) . )*? # the file name: anything but another " at " | |||||
| 861 | [ ] line [ ] \d+ \.? # " line 42." | |||||
| 862 | \n? \z # at the very end | |||||
| 863 | //x; | |||||
| 864 | 83 | 59 | chomp $text; | |||
| 865 | 83 | 107 | return $text; | |||
| 866 | 31 31 31 | 5261 21 88 | } | |||
| 867 | ||||||
| 868 | # _failure_reason | |||||
| 869 | # Purpose: Explain why an eval failed. An autodie exception carries | |||||
| 870 | # the operating system's reason; anything else (such as a | |||||
| 871 | # taint-mode "Insecure dependency") is reported as it is, | |||||
| 872 | # not replaced by a stale $! from some earlier call. | |||||
| 873 | # Entry Criteria: $error is the eval's $@. | |||||
| 874 | # Exit Status: Returns a one-line string. | |||||
| 875 | # Side Effects: None. A plain function, not a method. | |||||
| 876 | sub _failure_reason :Private { | |||||
| 877 | 13 | 8606 | my $error = shift; | |||
| 878 | ||||||
| 879 | 13 | 96 | my $reason = (blessed($error) && $error->can('errno') && length($error->errno())) ? $error->errno() : "$error"; | |||
| 880 | 13 | 51 | return _strip_location($reason); | |||
| 881 | 31 31 31 | 4712 10 75 | } | |||
| 882 | ||||||
| 883 | # _no_follow | |||||
| 884 | # Purpose: The O_NOFOLLOW open flag, or 0 where the OS lacks it. | |||||
| 885 | # Entry Criteria: None. | |||||
| 886 | # Exit Status: Returns an integer flag. | |||||
| 887 | # Side Effects: None. | |||||
| 888 | sub _no_follow :Private { | |||||
| 889 | 41 | 154 | return eval { Fcntl::O_NOFOLLOW() } || 0; | |||
| 890 | 31 31 31 | 3743 9 69 | } | |||
| 891 | ||||||
| 892 | # _report_fatal | |||||
| 893 | # Purpose: Tell the user why the program stopped. | |||||
| 894 | # Entry Criteria: $error is the exception from eval; $verbose is the | |||||
| 895 | # --verbose flag. | |||||
| 896 | # Exit Status: Returns the fatal exit status. | |||||
| 897 | # Side Effects: Prints to STDERR. | |||||
| 898 | sub _report_fatal :Private { | |||||
| 899 | 71 | 6227 | my ($class, $error, $verbose) = @_; | |||
| 900 | ||||||
| 901 | # Carp appends " at FILE line N."; that is noise for a command-line | |||||
| 902 | # user, but useful when debugging, so keep it with --verbose | |||||
| 903 | 71 | 125 | my $text = length($error // '') ? "$error" : 'Unknown error'; | |||
| 904 | 71 | 90 | $text = $verbose ? $text : _strip_location($text); | |||
| 905 | 71 | 42 | chomp $text; | |||
| 906 | ||||||
| 907 | # The reason may quote a hostile table name or file name: escape it | |||||
| 908 | 71 | 120 | print STDERR $class->_printable($class->i18n('fatal', { params => [$text] })), "\n"; | |||
| 909 | 71 | 84 | return $EXIT_FATAL; | |||
| 910 | 31 31 31 | 4458 15 77 | } | |||
| 911 | ||||||
| 912 | 1; | |||||
| 913 | ||||||