File Coverage

File:blib/lib/Database/BI/Controller/Dashboard.pm
Coverage:92.4%

linestmtbrancondsubtimecode
1package Database::BI::Controller::Dashboard;
2
3our $VERSION = '0.002.0';
4
5
12
12
12
10641
10
39
use Mojo::Base 'Mojolicious::Controller', -strict, -signatures;
6
7
12
12
12
7041
44
357
use Carp                qw(croak carp);
8
12
12
12
41
25
193
use Mojo::File;
9
12
12
12
24
15
207
use Mojo::JSON          qw(encode_json);
10
12
12
12
15
9
202
use Mojo::Util          qw(url_escape encode);
11
12
12
12
24
8
275
use File::Temp          qw(tempfile tempdir);
12
12
12
12
22
11
205
use Readonly;
13
12
12
12
21
17
247
use Socket              qw(inet_aton);
14
12
12
12
17
14
53
use Sub::Protected;
15
16# ---------------------------------------------------------------------------
17# Constants
18# ---------------------------------------------------------------------------
19
20# File extensions that Database::Abstraction can probe, in probe order.
21# Note: D::A uses ".sql" for SQLite -- NOT ".sqlite".
22Readonly my @SUPPORTED_EXT => qw( csv db sql xml psv );
23# Use \z (absolute end-of-string) not $ (which permits a trailing \n before \z).
24# A query param decoded from "file.csv%0A" has basename "sales.csv\n"; without \z
25# that passes the extension guard and reaches realpath with an embedded newline.
26Readonly my $EXT_RE        => do { my $p = join '|', @SUPPORTED_EXT; qr/\.(?:$p)\z/i };
27
28# Table name safe-identifier pattern -- must match DataSource's TABLE_NAME_RE
29# exactly: first character must be a letter or underscore (not a digit), so
30# that every table name the controller accepts can also be opened by DataSource
31# without an unhandled croak.
32Readonly my $TABLE_NAME_RE => qr/\A[A-Za-z_][A-Za-z0-9_]*\z/;
33
34# URL spec pattern shared by _open_spec and _spec_to_url.
35# qr{} delimiter avoids the \/ escaping noise required by the // form.
36# /s: makes . match \n so a percent-decoded newline inside a URL does not
37# silently truncate the captured URL before the \z end-of-string anchor.
38Readonly my $URL_SPEC_RE   => qr{\Aurl:(https?://.+)\z}si;
39
40# ---------------------------------------------------------------------------
41# I18N message dictionary for all user-visible strings in this controller.
42# To plug in a Locale::Maketext backend, extend _i18n() below.
43# ---------------------------------------------------------------------------
44
45Readonly my %MESSAGES => (
46        error_table_invalid    => 'Invalid table name "%s"',
47        error_table_open       => 'Could not open table "%s": %s',
48        error_file_open        => 'Could not open "%s": %s',
49        error_no_path          => 'No path specified',
50        error_not_found        => 'File or directory not found: %s',
51        error_dir_not_found    => 'Directory not found: %s',
52        error_write_failed     => 'Write failed: %s',
53        error_ext_required     => 'Use a .csv or .sql filename extension',
54        error_upload_none      => 'No file received',
55        error_upload_ext       => 'Unsupported file type. Accepted: CSV, PSV, XML, SQLite (.sql)',
56        error_upload_too_large => 'File too large (maximum %s MiB)',
57        error_path_required    => '"path" parameter is required',
58        error_url_required     => 'Please enter a URL',
59        error_url_invalid      => '"%s" is not a valid http:// or https:// URL',
60        error_url_fetch        => 'Could not load HTML table from "%s": %s',
61        error_url_ssrf         => '"%s" resolves to a private or reserved address and cannot be fetched',
62);
63
64# Maximum accepted upload body size.  Enforced both here (application layer)
65# and via Mojolicious max_request_size (transport layer) set in startup().
66# Must match $MAX_REQUEST_SIZE in BI.pm.
67Readonly my $MAX_UPLOAD_MIB   => 50;
68Readonly my $MAX_UPLOAD_BYTES => $MAX_UPLOAD_MIB * 1_048_576;
69
70# ---------------------------------------------------------------------------
71# Protected helpers
72# ---------------------------------------------------------------------------
73
74# _i18n($self, $key, @sprintf_args) -> $string
75#
76# Purpose: Look up a user-visible string from %MESSAGES and apply sprintf
77#          for positional arguments.  The entry point for future i18n
78#          backend integration (e.g. Locale::Maketext).
79# Entry:   $key must be a key in %MESSAGES; @args are sprintf positionals.
80# Exit:    Returns the formatted string, or a fallback containing $key.
81
78
78
78
78
78
3057
67
83
80
67
sub _i18n :Protected ($self, $key, @args) {
82
78
265
        my $tmpl = $MESSAGES{$key} // return "Internal error: unknown message key '$key'";
83
77
567
        return @args ? sprintf($tmpl, @args) : $tmpl;
84
12
12
12
3037
6
218
}
85
86# _is_safe_url($url) -> bool
87#
88# Purpose: SSRF guard.  Blocks the most dangerous classes of Server-Side Request
89#          Forgery targets: loopback aliases (localhost, 127/8) and literal
90#          private/link-local/CGNAT IPv4 addresses in the URL host component.
91#
92# Design rationale for scope:
93#   Resolving hostnames via DNS and checking the result is ineffective: a
94#   separate DNS lookup happens at LWP connect time (TOCTOU / DNS rebinding
95#   window).  The authoritative defence against hostname-based SSRF is a
96#   network-layer egress firewall.  This function handles the Perl-layer
97#   interception for the most common patterns that operators cannot easily
98#   filter at the network level: bare loopback aliases and literal private IPs
99#   hard-coded by an attacker.
100#
101# Blocked targets:
102#   localhost / 127.0.0.0/8 / 0.0.0.0 / ::1  -- loopback aliases
103#   10.0.0.0/8    -- RFC 1918 private (literal IP)
104#   172.16.0.0/12 -- RFC 1918 private (literal IP)
105#   192.168.0.0/16-- RFC 1918 private (literal IP)
106#   169.254.0.0/16-- link-local; AWS/GCP/Azure metadata endpoint (literal IP)
107#   100.64.0.0/10 -- CGNAT / Tailscale shared space (literal IP)
108#
109# Hostname-based targets (e.g. http://internal.corp.example.com/) are allowed
110# at this layer; block them with egress firewall rules instead.
111sub _is_safe_url {
112
95
28466
        my ($url) = @_;
113
95
279
        return 0 unless $url =~ m{\Ahttps?://([^/:?\[\]#]+)}i;
114
83
112
        my $host = lc $1;
115
116        # Block well-known loopback aliases.
117
83
325
        return 0 if $host eq 'localhost'
118                  || $host =~ /\A127\./
119                  || $host eq '0.0.0.0'
120                  || $host eq '::1';
121
122        # For literal IPv4 addresses only: check private/link-local/CGNAT ranges.
123        # Skipping DNS for hostname targets avoids a live network call in tests and
124        # removes the TOCTOU window that makes DNS-resolved checks illusory anyway.
125
62
151
        return 1 unless $host =~ /\A\d{1,3}(?:\.\d{1,3}){3}\z/;
126
127
47
198
        my $packed = inet_aton($host) or return 1;
128
46
70
        my $n = unpack 'N', $packed;
129
130
46
70
        return 0 if ($n & 0xFF000000) == 0x0A000000;        # 10/8
131
38
52
        return 0 if ($n & 0xFFF00000) == 0xAC100000;        # 172.16/12
132
31
47
        return 0 if ($n & 0xFFFF0000) == 0xC0A80000;        # 192.168/16
133
24
35
        return 0 if ($n & 0xFFFF0000) == 0xA9FE0000;        # 169.254/16 (link-local / metadata)
134
18
26
        return 0 if ($n & 0xFFC00000) == 0x64400000;        # 100.64/10 (CGNAT)
135
12
23
        return 1;
136}
137
138# _resolve_template($self) -> ($platform, $language)
139#
140# Purpose: Read platform and language from config, then resolve the Accept-Language
141#          header to a language code -- falling back to the config default when
142#          no template directory exists for the resolved language.
143# Exit:    Returns ($platform, $language) -- both guaranteed non-empty strings.
144
213
213
213
178
171
167
sub _resolve_template :Protected ($self) {
145
213
281
        my $conf     = $self->app->config;
146
213
1204
        my $platform = $conf->{platform} // 'web';
147
213
329
        my $default  = $conf->{language} // 'en';
148
213
361
        my $language = $self->_resolve_language($default);
149
213
289
        return ($platform, $language);
150
12
12
12
3770
10
104
}
151
152# _resolve_language($self, $default) -> $language_code
153#
154# Purpose: Extract the first two-letter language code from the Accept-Language
155#          request header, then validate that a template directory exists for
156#          that language.  Falls back to $default when the header is absent,
157#          unparseable, or points to a non-existent template directory.
158# Entry:   $default is a non-empty string (e.g. 'en').
159# Exit:    Returns a two-letter ISO 639-1 language code string.
160# Side Effects: Filesystem stat for the template directory.
161
221
221
221
221
11670
185
179
151
sub _resolve_language :Protected ($self, $default) {
162
221
314
        my $accept = $self->req->headers->accept_language // '';
163
221
2511
        my ($lang) = $accept =~ /
164                \b
165                ( [a-z]{2} )          # ISO 639-1 primary language subtag (exactly 2 lowercase letters)
166                (?: - [A-Z]{2} )?     # optional ISO 3166-1 region subtag: hyphen + 2 uppercase letters
167                \b
168        /x;
169
221
476
        $lang //= $default;
170
171        # Only use the resolved language if templates actually exist for it;
172        # prevents a 500 error when a browser sends e.g. Accept-Language: de
173        # but no templates/web/de/ directory is present.
174
221
319
        if ($lang ne $default) {
175
7
11
                my $platform = $self->app->config->{platform} // 'web';
176
7
51
                my $dir = $self->app->home->child("templates/$platform/$lang");
177
7
162
                $lang   = $default unless -d $dir;
178        }
179
221
341
        return $lang;
180
12
12
12
2626
12
98
}
181
182# _scan_data_dir($self) -> \@table_list
183#
184# Purpose: Return an arrayref of { name, file } hashrefs for every supported
185#          data file in the configured data_dir.  Used by the index action and
186#          by the join panel "available tables" dropdown.
187# Entry:   None.
188# Exit:    Returns [] when data_dir does not exist or contains no supported files.
189# Side Effects: Filesystem directory read.
190#
191# Optimisation: single-pass map replaces a two-pass grep+map chain, and calls
192# basename() only once per entry (the original chain called it twice: once in
193# grep to check the extension and again in map to extract the stem).
194# ->to_array avoids the extra flat-list expansion that ->each produced.
195
136
136
136
257
112
87
sub _scan_data_dir :Protected ($self) {
196
136
227
        my $dir = $self->app->home->child($self->app->config->{data_dir} // 'data');
197
136
3078
        return [] unless -d $dir;
198        return $dir->list->map(sub {
199
952
16688
                my $base = $_->basename;
200
952
11852
                return unless $base =~ $EXT_RE;
201
952
3246
                (my $name = $base) =~ s/\.[^.]+\z//;
202
952
1418
                { name => $name, file => $base }
203
136
1194
        })->to_array;
204
12
12
12
2211
11
87
}
205
206# _open_spec($self, $spec) -> ($DataSource, $label) or ()
207#
208# Purpose: Parse a "table:name" or "path:/abs/path" spec, verify the underlying
209#          file exists (Database::Abstraction creates empty objects silently for
210#          missing files, so we must check ourselves), and return a live
211#          DataSource object and a display label.
212# Entry:   $spec is a non-empty string.
213# Exit:    Returns ($datasource, $label) on success; an empty list on any failure
214#          (invalid spec format, file missing, DataSource init error).
215# Side Effects: Filesystem stat; DataSource construction (reads file header).
216
104
104
104
104
97
101
92
81
sub _open_spec :Protected ($self, $spec) {
217
104
318
        if ($spec =~ /\Atable:([A-Za-z0-9_]+)\z/) {
218
75
126
                my $table    = lc $1;
219
75
112
                my $data_dir = $self->app->home->child($self->app->config->{data_dir} // 'data');
220
75
375
1435
4228
                return () unless grep { -f $data_dir->child("$table.$_") } @SUPPORTED_EXT;
221
70
70
850
111
                my $src = eval { $self->open_table($table, directory => $data_dir->to_string) };
222
70
335
                return ($src, $table) if $src && !$@;
223        }
224        elsif ($spec =~ /\Apath:(.+)\z/) {
225
19
19
17
45
                my $file = eval { Mojo::File->new($1)->realpath };
226
19
455
                if (defined $file && -f $file && $file->basename =~ $EXT_RE) {
227
16
508
                        my $dir = $file->dirname->to_string;
228
16
341
                        (my $table = $file->basename) =~ s/\.[^.]+\z//;
229
16
16
173
43
                        my $src = eval { $self->open_table(lc $table, directory => $dir) };
230
16
68
                        return ($src, $file->basename) if $src && !$@;
231                }
232        }
233        elsif ($spec =~ $URL_SPEC_RE) {
234
2
20
                my $url = $1;
235
2
4
                return () unless _is_safe_url($url);
236
1
1
1
2
                my $src = eval { $self->open_table('', url => $url) };
237
1
5
                return ($src, $src->table_name) if $src && !$@;
238        }
239
11
144
        return ();
240
12
12
12
3832
12
88
}
241
242# _spec_to_url($self, $spec) -> $url_string
243#
244# Purpose: Convert a table/path spec back to a browseable URL (used for the
245#          back-link on join and export views).
246# Entry:   $spec is a "table:name" or "path:/abs" string.
247# Exit:    Returns a URL string beginning with '/'.
248
27
27
27
27
10642
25
40
18
sub _spec_to_url :Protected ($self, $spec) {
249
27
125
        return "/view/$1"                         if $spec =~ /\Atable:([A-Za-z0-9_]+)\z/;
250
11
36
        return '/open?path=' . url_escape($1)     if $spec =~ /\Apath:(.+)\z/;
251
6
21
        return '/import?url=' . url_escape($1)    if $spec =~ $URL_SPEC_RE;
252
3
13
        return '/';
253
12
12
12
1884
9
82
}
254
255# _get_columns($source, $records) -> @column_names
256#
257# Purpose: Return an ordered column list from a DataSource object.
258#          For CSV/PSV the DataSource stores the original file-header order.
259#          For SQLite/XML where no file-header order is available, the fallback
260#          is: id_column first (only if it actually appears in the data), then
261#          the remaining columns sorted alphabetically.
262# Entry:   $source is a DataSource; $records is an arrayref of hashrefs (may be empty).
263# Exit:    Returns a flat list of column-name strings (may be empty).
264#
265# Reduction: columns() is a pure accessor -- cache once to avoid a redundant
266# method dispatch on the second reference.
267sub _get_columns {
268
180
10067
        my ($source, $records) = @_;
269
180
339
        my $cols = $source->columns;
270
180
436
        return @$cols if $cols;
271
26
45
        return () unless $records->[0];
272
24
66
24
20
76
68
        my %all = map { $_ => 1 } keys %{ $records->[0] };
273
24
53
        my $c   = $source->id_column;
274
24
89
        my $id  = ($c && $all{$c}) ? $c : (sort keys %all)[0];
275
24
31
        delete $all{$id};
276
24
56
        return ($id, sort keys %all);
277}
278
279# _apply_filter_spec($records, $spec) -> \@filtered_records
280#
281# Purpose: Apply one "col:op:val" filter to an arrayref of record hashrefs and
282#          return a new (possibly shorter) arrayref.  The original is not mutated.
283#          Value may contain colons: "col:op:val:with:colons" -- split limit 3
284#          puts everything after the second colon into $val.
285#          Unknown operators fall through to 1 (all rows pass) so new operators
286#          added in future do not break existing callers.
287# Entry:   $records is an arrayref of hashrefs; $spec is a colon-delimited string.
288# Exit:    Returns filtered arrayref (same reference if spec is invalid/unparseable).
289#
290# Domain Constraints (operator names):
291#   Operator names are CASE-SENSITIVE.  Only the exact lowercase forms below are
292#   recognised; an uppercase or mixed-case name (e.g. "EQ", "Lt") falls through
293#   to the default `1` branch and all rows pass unfiltered.
294#   Valid operators:
295#     String (value comparison is case-insensitive via lc()):
296#       eq, ne, contains, starts
297#     Numeric (Perl <, <=, >, >= coercion):
298#       lt (strict), le (inclusive), gt (strict), ge (inclusive)
299#     Unary (val is ignored):
300#       empty (cell eq ''), notempty (cell ne '')
301#   Boundary rules for numeric operators:
302#     le/ge include the boundary value; lt/gt exclude it.
303#   contains/starts with empty val ('') match every record (index always 0).
304#
305# Reduction: split(/:/, $spec, 3) always produces at least one defined element,
306# so "defined $col" is structurally guaranteed true and is removed.
307# "$_->{$col} // ''" replaces the equivalent ternary form.
308#
309# Optimisation: $lval = lc($val) is computed once before the grep.  Without this
310# precompute, every case-insensitive operator (eq, ne, contains, starts) would
311# call lc($val) O(N) times for N records -- a pure O(N) waste since $val is
312# constant within a single filter application.
313sub _apply_filter_spec {
314
277
74754
        my ($records, $spec) = @_;
315
277
362
        my ($col, $op, $val) = split /:/, $spec, 3;
316
277
669
        return $records unless length($col // '') && defined $op && length $op;
317
269
219
        $val //= '';
318
269
215
        my $lval = lc $val;     # precompute once; avoids O(N) redundant lc() inside grep
319        return [grep {
320
269
758
250
745
                my $cell = $_->{$col} // '';
321
758
2135
                $op eq 'eq'       ? lc($cell) eq $lval                :
322                $op eq 'ne'       ? lc($cell) ne $lval                :
323                $op eq 'contains' ? index(lc($cell), $lval) != -1     :
324                $op eq 'starts'   ? index(lc($cell), $lval) == 0      :
325                $op eq 'lt'       ? $cell <  $val                     :
326                $op eq 'le'       ? $cell <= $val                     :
327                $op eq 'gt'       ? $cell >  $val                     :
328                $op eq 'ge'       ? $cell >= $val                     :
329                $op eq 'empty'    ? $cell eq ''                        :
330                $op eq 'notempty' ? $cell ne ''                        :
331                1
332        } @$records];
333}
334
335# _apply_filters($self, $records) -> ($filtered_records, \@raw_specs, $json_string)
336#
337# Purpose: Read all "f=" query params, apply them in order via _apply_filter_spec,
338#          and return: the filtered arrayref, the raw spec strings (for building
339#          export URLs), and a script-safe JSON string for pre-populating the UI.
340# Entry:   $records is an arrayref of hashrefs.
341# Exit:    Three-element list; never croaks.  $json_string has '</' escaped to
342#          '<\/' so it is safe to embed directly in a <script> block.
343#
344# Reduction: the original code had two sequential loops over @specs -- one to
345# apply filters, one to parse them for JSON.  By Modus Ponens, _apply_filter_spec
346# returns $records unchanged for a malformed spec (same guard condition), so
347# skipping the call for malformed specs is equivalent.  Merging into one loop
348# eliminates a full O(n) pass and the redundant split() on every spec.
349
95
95
95
95
90
83
72
74
sub _apply_filters :Protected ($self, $records) {
350
95
95
78
237
        my @specs  = @{ $self->every_param('f') // [] };
351
95
10793
        my @parsed;
352
95
121
        for my $s (@specs) {
353
103
191
                my ($col, $op, $val) = split /:/, $s, 3;
354
103
324
                next unless length($col // '') && defined $op && length $op;
355
99
228
                push @parsed, { col => $col, op => $op, val => $val // '' };
356
99
111
                $records = _apply_filter_spec($records, $s);
357        }
358
95
243
        my $json = encode_json(\@parsed);
359
95
626
        $json =~ s{</}{<\\/}g;
360
95
197
        return ($records, \@specs, $json);
361
12
12
12
4540
11
99
}
362
363# _left_join($left_recs, $left_cols, $left_key,
364#            $right_recs, $right_cols, $right_key, $right_label)
365#   -> (\@merged_records, \@merged_columns)
366#
367# Purpose: Perform a single in-memory left join.  Every left row is kept.
368#          Right-table columns are appended for rows that match on the join key.
369#          Unmatched rows receive undef for right-table columns.
370#          If a right column name collides with a left column (other than the
371#          join key itself), the right column is prefixed with "$right_label.".
372# Entry:   All arrayref args non-undef; key strings non-empty; $right_label is
373#          a display label used for collision-prefix (not for SQL quoting).
374# Exit:    Returns (\@merged, \@column_list).
375# Side Effects: None; allocates new record hashrefs.
376sub _left_join {
377
21
14333
        my ($left_recs, $left_cols, $left_key,
378            $right_recs, $right_cols, $right_key, $right_label) = @_;
379
380        # Build a lookup hash from join key to first matching right row.
381
21
20
        my %right_idx;
382
21
29
        for my $row (@$right_recs) {
383
55
65
                my $k = $row->{$right_key} // '';
384
55
102
                $right_idx{$k} //= $row;
385        }
386
387        # Map right column names: drop the join key (redundant), prefix collisions.
388
21
76
28
75
        my %left_set = map { $_ => 1 } @$left_cols;
389
21
24
        my (@add_cols, %col_map);
390
21
59
27
55
        for my $col (grep { $_ ne $right_key } @$right_cols) {
391
38
50
                my $out = $left_set{$col} ? "${right_label}.${col}" : $col;
392
38
35
                $col_map{$col} = $out;
393
38
39
                push @add_cols, $out;
394        }
395
396        # Precompute [$right_col, $mapped_col] pairs once before the merge loop.
397        # Without this, "grep { $_ ne $right_key } @$right_cols" would run on
398        # every left row -- O(N_left * R) grep iterations for R right columns.
399        # Precomputing reduces that to a single O(R) pass.
400
38
52
        my @rcols = map { [$_, $col_map{$_}] }
401
21
59
24
49
                    grep { $_ ne $right_key } @$right_cols;
402
403
21
17
        my @merged;
404
21
24
        for my $left_row (@$left_recs) {
405
76
80
                my $k         = $left_row->{$left_key} // '';
406
76
93
                my $right_row = $right_idx{$k} // {};
407
76
121
                my %row       = %$left_row;
408
76
145
                $row{ $_->[1] } = $right_row->{ $_->[0] } for @rcols;
409
76
67
                push @merged, \%row;
410        }
411
412
21
66
        return (\@merged, [@$left_cols, @add_cols]);
413}
414
415# _combine_tables(\@sources) -> (\@merged_records, \@merged_columns)
416#
417# Purpose: Perform a vertical stack (UNION ALL equivalent) of two or more
418#          record sets.  The merged column list is the union of all source
419#          column lists: the first source's columns appear first (in their
420#          original order), then each subsequent source contributes any new
421#          column names it introduces, in order.  Where a source does not
422#          have a particular column, its rows receive an empty string for
423#          that column.
424# Entry:   $sources is an arrayref of [$records_aref, $columns_aref] pairs.
425#          At least one element is required; a single-element call returns
426#          that element's data unchanged (no allocation beyond the column copy).
427# Exit:    Returns (\@merged_records, \@merged_columns).
428# Side Effects: None; allocates new record hashrefs (original rows are not mutated).
429sub _combine_tables {
430
4
5
        my ($sources) = @_;
431
432        # Build the unified column order: first source's columns first, then any
433        # new columns introduced by each subsequent source in their file order.
434
4
5
        my (@all_cols, %seen);
435
4
4
        for my $src (@$sources) {
436
8
8
7
7
                for my $col (@{ $src->[1] }) {
437
52
52
                        push @all_cols, $col unless $seen{$col}++;
438                }
439        }
440
441
4
3
        my @merged;
442
4
5
        for my $src (@$sources) {
443
8
8
                my ($recs, $cols) = @$src;
444
8
52
8
42
                my %has_col = map { $_ => 1 } @$cols;
445
8
10
                for my $row (@$recs) {
446
96
49
                        my %new_row;
447                        $new_row{$_} = $has_col{$_} ? ($row->{$_} // '') : ''
448
96
378
                                for @all_cols;
449
96
74
                        push @merged, \%new_row;
450                }
451        }
452
453
4
7
        return (\@merged, \@all_cols);
454}
455
456# _csv_row(@fields) -> $csv_line_with_crlf
457#
458# Purpose: Format one row as an RFC 4180 CSV line.  Fields containing commas,
459#          double-quotes, or newlines are enclosed in double-quotes; embedded
460#          double-quotes are doubled.  The line ends with CRLF.
461# Entry:   @fields may contain undef (treated as empty string).
462# Exit:    Returns a string ending with "\r\n".
463sub _csv_row {
464        return join(',', map {
465
309
1701
16614
1213
                my $f = $_ // '';
466
1701
12
12
1787
16
24
                $f =~ /[,"\r\n]/ ? do { (my $q = $f) =~ s/"/""/g; qq{"$q"} } : $f;
467        } @_) . "\r\n";
468}
469
470# _build_export_url($self, $left_spec, \@join_specs, \@filter_specs) -> $url_string
471#
472# Purpose: Assemble a /export?l=...&j=...&f=... URL string for the current
473#          logical view.  The caller must apply | html in TT to escape & as &amp;.
474# Entry:   All args non-undef; spec arrays may be empty.
475# Exit:    Returns a relative URL string beginning with '/export'.
476
99
99
99
99
99
99
99
6598
118
88
74
81
91
72
sub _build_export_url :Protected ($self, $left_spec, $join_specs, $filter_specs, $combine_specs = undef) {
477
99
162
        my $u = '/export?l=' . url_escape($left_spec);
478
99
1182
        $u .= '&j=' . url_escape($_) for @$join_specs;
479
99
99
219
228
        $u .= '&c=' . url_escape($_) for @{ $combine_specs // [] };
480
99
162
        $u .= '&f=' . url_escape($_) for @$filter_specs;
481
99
851
        return $u;
482
12
12
12
6287
12
105
}
483
484# _list_dir($self, $dir, $want_files) -> { dirs => [...], files => [...] }
485#
486# Purpose: Shared directory-listing kernel used by browse (which needs files)
487#          and dirs_api (which needs only subdirectories).  Hidden entries
488#          (names starting with ".") are excluded.  Entries are sorted
489#          case-insensitively.
490# Entry:   $dir is a Mojo::File pointing to an existing readable directory.
491#          $want_files: if true, populate 'files' with entries matching $EXT_RE.
492# Exit:    Returns a hashref with 'dirs' and 'files' arrayrefs (files is always
493#          present but empty when $want_files is false).
494# Side Effects: Filesystem directory read.
495
21
21
21
21
21
9576
21
21
20
15
sub _list_dir :Protected ($self, $dir, $want_files) {
496
21
23
        my (@dirs, @files);
497
21
64
        if (opendir my $dh, $dir->to_string) {
498
20
600
                while (my $entry = readdir $dh) {
499
182
620
                        next if $entry eq '.' || $entry eq '..' || $entry =~ /\A\./;
500
83
81
                        my $f = $dir->child($entry);
501
83
549
                        if (-d $f) {
502
53
317
                                push @dirs,  { name => $entry, path => $f->to_string };
503                        }
504                        elsif ($want_files && $entry =~ $EXT_RE) {
505
15
162
                                push @files, { name => $entry, path => $f->to_string };
506                        }
507                }
508
20
131
                closedir $dh;
509        }
510        return {
511
113
126
                dirs  => [ sort { lc($a->{name}) cmp lc($b->{name}) } @dirs  ],
512
21
11
97
21
                files => [ sort { lc($a->{name}) cmp lc($b->{name}) } @files ],
513        };
514
12
12
12
3047
11
89
}
515
516# _write_sqlite_db($self, $records, $columns) -> $raw_bytes
517#
518# Purpose: Write $records to an in-process SQLite database (single table named
519#          "data"; every column declared TEXT) and return the raw file bytes.
520#          The temporary file is created, written, read, and then unlinked in
521#          one synchronous sequence.  Used by both _render_sqlite (HTTP download)
522#          and export_write (filesystem write) to eliminate code duplication.
523# Entry:   $records is an arrayref of hashrefs; $columns is an arrayref of names.
524# Exit:    Returns scalar binary string.  Croaks on any DBI error.
525# Side Effects: Creates and unlinks one temporary file under system tmpdir.
526
24
24
24
24
24
30983
25
20
36
18
sub _write_sqlite_db :Protected ($self, $records, $columns) {
527        # D~ fix: an empty column list produces "CREATE TABLE data ()" which is
528        # invalid SQLite syntax.  Croak before touching the filesystem.
529
24
67
        croak 'Cannot export SQLite: result set has no columns'
530                unless @$columns;
531
532
21
69
        require DBI;
533
21
56
        my ($tmp_fh, $tmpfile) = tempfile(SUFFIX => '.db', UNLINK => 0);
534
21
4128
        close $tmp_fh;
535
536
21
21
21
107
        my $dbh = eval { DBI->connect("dbi:SQLite:dbname=$tmpfile", '', '', {
537                RaiseError => 1, AutoCommit => 1,
538        }) };
539        # Guard both the eval-caught die ($@) and the undef-without-die edge case.
540
21
14560
        if ($@ || !$dbh) {
541
2
47
                unlink $tmpfile;
542
2
24
                croak "DBI connect failed: $@";
543        }
544
545        # O~ fix: wrap all DBI work in eval so any mid-flight exception still
546        # triggers cleanup (disconnect + unlink) before the error propagates.
547
19
26
        eval {
548
19
74
74
33
82
132
                my @quoted = map { (my $c = $_) =~ s/"/""/g; qq{"$c"} } @$columns;
549
19
74
32
130
                $dbh->do('CREATE TABLE "data" (' . join(', ', map { "$_ TEXT" } @quoted) . ')');
550
17
24016
                if (@$records) {
551
15
63
                        my $ph  = join(', ', ('?') x scalar @$columns);
552
15
103
                        my $sth = $dbh->prepare(
553                                'INSERT INTO "data" (' . join(', ', @quoted) . ") VALUES ($ph)"
554                        );
555
15
612
                        for my $row (@$records) {
556
61
309
2878
38099
                                $sth->execute(map { $row->{$_} } @$columns);
557                        }
558                }
559
17
1825
                $dbh->disconnect;
560        };
561
19
53
        if (my $err = $@) {
562
2
2
2
35
                eval { $dbh->disconnect };   # best-effort; may already be disconnected
563
2
46
                unlink $tmpfile;
564
2
29
                croak $err;
565        }
566
567
17
81
        my $data = Mojo::File->new($tmpfile)->slurp;
568
17
1447
        unlink $tmpfile;
569
17
425
        return $data;
570
12
12
12
4225
12
93
}
571
572# _run_export_pipeline($self) -> ($records, \@columns, $left_label) or ()
573#
574# Purpose: Shared join+filter pipeline executed by both export_data (GET, download)
575#          and export_write (POST, filesystem write).  Opens the left table, applies
576#          all join steps, then applies all filter specs.
577# Entry:   Reads "l=", "j=" (repeatable), and "f=" (repeatable) query/body params.
578# Exit:    On success: ($filtered_arrayref, \@column_names, $left_label_string).
579#          On failure: empty list (left table not found / fetch error).
580# Side Effects: Opens data files; may emit carp warnings for empty result sets.
581
62
62
62
60
53
49
sub _run_export_pipeline :Protected ($self) {
582
62
99
        my $left_spec = $self->param('l') // '';
583
62
6043
        my ($left_src, $left_label) = $self->_open_spec($left_spec);
584
62
275
        return () unless $left_src;
585
586
54
54
49
124
        my $left_recs = eval { $left_src->fetch_all };
587
54
87
        return () if $@;
588
54
83
        $left_recs //= [];
589
590
54
92
        my @columns = _get_columns($left_src, $left_recs);
591
54
58
        my $records  = $left_recs;
592
593
54
54
47
105
        for my $jspec (@{ $self->every_param('j') }) {
594
3
112
                my ($right_spec, $left_key, $right_key) = split /\|/, $jspec, 3;
595
3
14
                next unless defined $right_spec && defined $left_key && defined $right_key;
596
3
16
4
17
                my %col_set = map { $_ => 1 } @columns;
597
3
8
                next unless $col_set{$left_key};
598
3
6
                my ($right_src, $right_label) = $self->_open_spec($right_spec);
599
3
37
                next unless $right_src;
600
3
3
4
5
                my $right_recs = eval { $right_src->fetch_all } // [];
601
3
4
                next if $@;
602
3
5
                my @right_cols = _get_columns($right_src, $right_recs);
603
3
9
3
10
                my %right_set  = map { $_ => 1 } @right_cols;
604
3
9
                next unless $right_set{$right_key};
605
3
8
                ($records, my $new_cols) = _left_join(
606                        $records, \@columns, $left_key,
607                        $right_recs, \@right_cols, $right_key, $right_label,
608                );
609
3
13
                @columns = @$new_cols;
610        }
611
612        # Combine (vertical stack) with any c= sources.  Runs after joins so a join
613        # result can itself be stacked with another table in a single pipeline.
614
54
54
1944
69
        my @cspecs = @{ $self->every_param('c') // [] };
615
54
1287
        if (@cspecs) {
616
1
2
                my @sources = ([$records, \@columns]);
617
1
1
                for my $cspec (@cspecs) {
618
1
2
                        my ($csrc) = $self->_open_spec($cspec);
619
1
3
                        next unless $csrc;
620
1
1
1
2
                        my $crecs = eval { $csrc->fetch_all } // [];
621
1
2
                        next if $@;
622
1
2
                        my @ccols = _get_columns($csrc, $crecs);
623
1
4
                        push @sources, [$crecs, \@ccols];
624                }
625
1
50
                if (@sources > 1) {
626
1
3
                        ($records, my $new_cols) = _combine_tables(\@sources);
627
1
6
                        @columns = @$new_cols;
628                }
629        }
630
631
54
54
47
70
        for my $s (@{ $self->every_param('f') }) {
632
10
178
                $records = _apply_filter_spec($records, $s);
633        }
634
635
54
1051
        return ($records, \@columns, $left_label);
636
12
12
12
4458
10
89
}
637
638# _serialize_csv($records, \@columns) -> $utf8_string
639#
640# Purpose: Build a complete RFC 4180 CSV document (header + data rows) from
641#          $records.  Shared by _render_csv (HTTP download) and the csv branch
642#          of export_write (filesystem write) to eliminate duplicated logic.
643# Entry:   $records is an arrayref of hashrefs; $columns is an arrayref of names.
644# Exit:    Returns a UTF-8 Perl string ending with CRLF.
645#
646# Optimisation: rows are collected in @lines and joined in one operation.
647# The previous pattern ($out .= _csv_row(...) per row) triggers O(N) string
648# reallocations; joining a pre-built array is a single O(total_bytes) allocation.
649# For exports of thousands of rows the difference is measurable.
650#
651# XS note: for very large exports, replacing _csv_row with Text::CSV_XS
652# (available from CPAN) would further reduce serialisation time -- its C-level
653# implementation is roughly 5-10x faster than the pure-Perl version here.
654sub _serialize_csv {
655
50
9123
        my ($records, $columns) = @_;
656
50
95
        my @lines = _csv_row(@$columns);
657
50
72
        for my $row (@$records) {
658
239
1399
188
1087
                push @lines, _csv_row(map { $row->{$_} } @$columns);
659        }
660
50
143
        return encode('UTF-8', join('', @lines));
661}
662
663# _render_csv($self, $records, \@columns, $name) -> void (renders response)
664#
665# Purpose: Serialise $records to RFC 4180 CSV and emit as a UTF-8 download.
666# Entry:   $name is a safe filename stem (alphanumeric/underscore).
667# Exit:    Renders the Mojolicious response; does not return a meaningful value.
668# Side Effects: Writes HTTP response headers and body.
669
27
27
27
27
27
27
28
27
28
21
25
21
sub _render_csv :Protected ($self, $records, $columns, $name) {
670
27
45
        $self->res->headers->content_type('text/csv; charset=UTF-8');
671
27
316
        $self->res->headers->content_disposition(qq{attachment; filename="${name}.csv"});
672
27
273
        $self->render(data => _serialize_csv($records, $columns));
673
12
12
12
2375
7
86
}
674
675# _render_sqlite($self, $records, \@columns, $name) -> void (renders response)
676#
677# Purpose: Write $records to a SQLite database and emit it as a binary download.
678# Entry:   $name is a safe filename stem.
679# Exit:    Renders the Mojolicious response.
680# Side Effects: Creates and unlinks a temporary file; writes HTTP response.
681
6
6
6
6
6
6
7
7
8
5
6
6
sub _render_sqlite :Protected ($self, $records, $columns, $name) {
682
6
19
        my $data = $self->_write_sqlite_db($records, $columns);
683
6
21
        $self->res->headers->content_type('application/vnd.sqlite3');
684
6
120
        $self->res->headers->content_disposition(qq{attachment; filename="${name}.db"});
685
6
71
        $self->render(data => $data);
686
12
12
12
1607
43
95
}
687
688# ---------------------------------------------------------------------------
689# Public actions
690# ---------------------------------------------------------------------------
691
692 - 727
=head1 ACTIONS

=head2 index

C<GET /> -- Scan C<data_dir> and present a card grid of available tables.

=head3 API SPECIFICATION

=head4 INPUT

None (reads C<data_dir> from application config).

=head4 OUTPUT

Renders C<home.html.tt> with:

  tables     ARRAYREF of { name => $stem, file => $basename }
  title      'Choose a Database'

=head3 MESSAGES

None produced by this action; any file-scan errors are silently ignored
(empty directory yields an empty card grid).

=head3 FORMAL SPECIFICATION

  index == lambda self .
    let dir  = resolve self.app.config.data_dir in
    let tbls = { b | b in dir /\ basename(b) =~ EXT_RE } in
    post render(home, tables: map(stem, tbls))

=head3 EXAMPLE

  GET / -> 200 text/html containing a card for each file in data/

=cut
728
729
16
16
16
118736
19
11
sub index ($self) {
730
16
45
        my ($platform, $language) = $self->_resolve_template;
731
732
16
44
        $self->render(
733                template => "$platform/$language/home",
734                handler  => 'tt',
735                format   => 'html',
736                tables   => $self->_scan_data_dir,
737                title    => 'Choose a Database',
738        );
739}
740
741 - 804
=head2 view

C<GET /view/:table> -- Open and display the chosen table from C<data_dir>.

=head3 API SPECIFICATION

=head4 INPUT

  :table   string   Table name (alphanumeric + underscore).  Returns 404 on
                    other characters.
  ?f=      string   (repeatable) Filter spec: "col:op:val".

=head4 DOMAIN CONSTRAINTS: :table

C<:table> is validated against C<\A[A-Za-z_][A-Za-z0-9_]*\z> before C<open_table>
is ever called.  The first character must be a letter (A-Z, a-z) or an
underscore; subsequent characters may also include digits (0-9).

=over 4

=item Valid partition

C<sales> (letters only), C<_temp> (underscore-start), C<report_2024>
(mixed).  A name that is valid but has no backing file returns 200 with
an error message -- NOT 404.

=item Invalid partition

C<1sales> (digit-start, 404), C<my.data> (dot, 404), C<my-data>
(hyphen, 404), non-ASCII characters (404).

=item Boundary values

C<a> (single letter, valid), C<_> (single underscore, valid), C<1>
(single digit, 404), C<a1> (letter then digit, valid), C<1a> (digit
then letter, 404).

=back

=head4 OUTPUT

On success renders C<dashboard.html.tt> with table data.
On error re-renders C<home.html.tt> with an C<error> stash variable.

=head3 MESSAGES

  error_table_open   -- DataSource initialisation or fetch_all threw.

=head3 FORMAL SPECIFICATION

  view == lambda self .
    pre  self.stash('table') =~ TABLE_NAME_RE
    let src     = open_table(table) in
    let records = src.fetch_all()   in
    let cols    = get_columns(src, records) in
    post render(dashboard, records: filter(records, f_params))

=head3 EXAMPLE

  GET /view/sales      -> 200 HTML table of all sales rows
  GET /view/sales?f=region:eq:North -> shows only North region rows
  GET /view/../etc     -> 404

=cut
805
806
75
75
75
897418
82
63
sub view ($self) {
807
75
146
        my ($platform, $language) = $self->_resolve_template;
808
75
106
        my $table = $self->stash('table');
809
810
75
580
        unless (defined $table && $table =~ $TABLE_NAME_RE) {
811
14
102
                return $self->reply->not_found;
812        }
813
814
61
353
        my ($source, $records);
815
61
61
61
56
145
141
        eval { $source = $self->open_table($table); $records = $source->fetch_all };
816
61
3543
        if ($@) {
817
9
52
                return $self->render(
818                        template => "$platform/$language/home",
819                        handler  => 'tt',
820                        format   => 'html',
821                        tables   => [],
822                        title    => 'Choose a Database',
823                        error    => $self->_i18n('error_table_open', $table, $@),
824                );
825        }
826
827
52
124
        my @columns = _get_columns($source, $records);
828
52
128
        my ($filtered, $filter_specs, $filters_json) = $self->_apply_filters($records);
829
830
52
229
        $self->render(
831                template         => "$platform/$language/dashboard",
832                handler          => 'tt',
833                format           => 'html',
834                records          => $filtered,
835                columns          => \@columns,
836                table            => $table,
837                title            => ucfirst($table),
838                left_spec        => "table:$table",
839                combine_specs    => [],
840                current_joins    => [],
841                available_tables => $self->_scan_data_dir,
842                join_summaries   => [],
843                filter_specs     => $filter_specs,
844                filters_json     => $filters_json,
845                export_url       => $self->_build_export_url("table:$table", [], $filter_specs),
846        );
847}
848
849 - 888
=head2 browse

C<GET /browse> -- Navigate the filesystem and pick a data file.

=head3 API SPECIFICATION

=head4 INPUT

  ?path=   string   Absolute filesystem path to browse (default: $HOME).
                    Returns 404 when the path does not exist or is not a
                    directory.

=head4 OUTPUT

Renders C<browse.html.tt> with:

  current_path   string
  parent_href    string or undef (undef at filesystem root)
  crumbs         ARRAYREF of { name, href }
  dirs           ARRAYREF of { name, href }   -- subdirectories, sorted
  files          ARRAYREF of { name, href }   -- supported data files, sorted

=head3 MESSAGES

None; errors render as 404.

=head3 FORMAL SPECIFICATION

  browse == lambda self .
    let dir = realpath(param('path') // HOME) in
    pre  is_dir(dir)
    post render(browse, dirs: subdirs(dir), files: supported_files(dir))

=head3 EXAMPLE

  GET /browse                            -> 200, lists $HOME
  GET /browse?path=/tmp                  -> 200, lists /tmp
  GET /browse?path=/nonexistent/xyz      -> 404

=cut
889
890
16
16
16
138067
16
14
sub browse ($self) {
891
16
32
        my ($platform, $language) = $self->_resolve_template;
892
893
16
40
        my $raw_path = $self->param('path') // $ENV{HOME} // '/';
894
16
16
1894
25
        my $dir      = eval { Mojo::File->new($raw_path)->realpath };
895
16
1553
        return $self->reply->not_found unless defined $dir && -d $dir;
896
897
9
92
        my $listing = $self->_list_dir($dir, 1);
898
899        # Add browse hrefs to file entries (dirs already have path; browse needs href).
900        my @dirs = map {
901
13
92
                +{ name => $_->{name}, href => '/browse?path=' . url_escape($_->{path}) }
902
9
9
12
14
        } @{ $listing->{dirs} };
903
904        my @files = map {
905
12
100
                +{ name => $_->{name}, href => '/open?path=' . url_escape($_->{path}) }
906
9
9
39
12
        } @{ $listing->{files} };
907
908        # Build breadcrumb trail from filesystem root down to $dir.
909
9
67
        my @crumbs;
910        {
911
9
9
8
9
                my $f = $dir;
912
9
8
                while (1) {
913
35
43
                        my $name = $f->basename;
914
35
438
                        $name    = '/' unless length $name;
915
35
44
                        unshift @crumbs, { name => $name, href => '/browse?path=' . url_escape($f->to_string) };
916
35
304
                        my $parent = $f->dirname;
917
35
487
                        last if $parent->to_string eq $f->to_string;
918
26
95
                        $f = $parent;
919                }
920        }
921
922
9
56
        my $parent      = $dir->dirname;
923
9
98
        my $parent_href = $parent->to_string ne $dir->to_string
924                ? '/browse?path=' . url_escape($parent->to_string)
925                : undef;
926
927
9
108
        $self->render(
928                template     => "$platform/$language/browse",
929                handler      => 'tt',
930                format       => 'html',
931                title        => 'Browse Files',
932                current_path => $dir->to_string,
933                parent_href  => $parent_href,
934                crumbs       => \@crumbs,
935                dirs         => \@dirs,
936                files        => \@files,
937        );
938}
939
940 - 1010
=head2 open_file

C<GET /open> -- Open a supported data file from any absolute filesystem path.

=head3 API SPECIFICATION

=head4 INPUT

  ?path=   string   Absolute path to the data file.  Returns 404 when missing,
                    not a regular file, or the extension is not in SUPPORTED_EXT.
  ?f=      string   (repeatable) Filter spec: "col:op:val".

=head4 DOMAIN CONSTRAINTS: ?path=

=over 4

=item Extension filter (C<EXT_RE>)

The basename must match C<\.(?:csv|db|sql|xml|psv)\z> (case-insensitive).
The C<\z> anchor (absolute end-of-string) means a URL-encoded trailing
newline (e.g. C<file.csv%0A> decoded to C<file.csv\n>) does NOT pass --
the C<\n> falls after the C<\z> boundary and the extension check fails.

=item Valid partition

C</data/sales.csv> (lowercase extension), C</tmp/REPORT.CSV> (uppercase
extension, /i matches), any C<.db>, C<.sql>, C<.xml>, C<.psv> regular
file.

=item Invalid partition

Absent C<?path=> (404), non-existent file (404), directory instead of
file (404), unsupported extension such as C<.txt> (404), empty string
(404), path containing a C<%0A> (newline) suffix (404).

=item Double-extension filenames

A file like C<file.php.csv> passes C<EXT_RE> (last segment is C<.csv>)
but produces table stem C<file.php> which fails C<DataSource>'s
C<TABLE_NAME_RE>.  The C<open_table> call is inside C<eval>, so the
croak is caught and the action returns 200 with a friendly error, not
a 500.

=back

=head4 OUTPUT

On success renders C<dashboard.html.tt> with C<back_url> pointing to the
directory's C</browse> page, and C<file_path> set to the resolved absolute
path (used by the template to record the file in C<localStorage>).
On error re-renders C<home.html.tt> with C<error>, C<back_url>, C<back_label>.

=head3 MESSAGES

  error_file_open   -- DataSource threw during initialisation or fetch_all.

=head3 FORMAL SPECIFICATION

  open_file == lambda self .
    let file = realpath(param('path')) in
    pre  is_file(file) /\ basename(file) =~ EXT_RE
    let src = open_table(stem(file), dir=dirname(file)) in
    post render(dashboard, file_path: file.to_string)

=head3 EXAMPLE

  GET /open?path=/data/archive/sales.csv   -> 200, table view
  GET /open                                -> 404
  GET /open?path=/etc/passwd               -> 404

=cut
1011
1012
53
53
53
485184
58
49
sub open_file ($self) {
1013
53
144
        my ($platform, $language) = $self->_resolve_template;
1014
53
105
        my $file_path = $self->param('path');
1015
1016
53
6574
        return $self->reply->not_found unless defined $file_path;
1017
1018
48
48
42
81
        my $file = eval { Mojo::File->new($file_path)->realpath };
1019
48
2657
        return $self->reply->not_found
1020                unless defined $file && -f $file && $file->basename =~ $EXT_RE;
1021
1022
31
1228
        my $dir      = $file->dirname;
1023
31
639
        (my $table   = $file->basename) =~ s/\.[^.]+\z//;
1024
31
405
        $table       = lc $table;
1025
31
47
        my $back     = '/browse?path=' . url_escape($dir->to_string);
1026
31
449
        my $filename = $file->basename;
1027
31
317
        my $lspec    = 'path:' . $file->to_string;
1028
1029
31
80
        my ($source, $records);
1030
31
31
26
26
48
60
        eval { $source = $self->open_table($table, directory => $dir->to_string); $records = $source->fetch_all };
1031
31
4150
        if ($@) {
1032
9
54
                return $self->render(
1033                        template   => "$platform/$language/home",
1034                        handler    => 'tt',
1035                        format     => 'html',
1036                        tables     => [],
1037                        title      => 'Error',
1038                        error      => $self->_i18n('error_file_open', $filename, $@),
1039                        back_url   => $back,
1040                        back_label => 'Back to browser',
1041                );
1042        }
1043
1044
22
48
        my @columns  = _get_columns($source, $records);
1045
22
63
        my ($filtered, $filter_specs, $filters_json) = $self->_apply_filters($records);
1046
1047
22
88
        $self->render(
1048                template         => "$platform/$language/dashboard",
1049                handler          => 'tt',
1050                format           => 'html',
1051                records          => $filtered,
1052                columns          => \@columns,
1053                table            => $table,
1054                title            => $filename,
1055                back_url         => $back,
1056                back_label       => 'Back to browser',
1057                file_path        => $file->to_string,
1058                left_spec        => $lspec,
1059                combine_specs    => [],
1060                current_joins    => [],
1061                available_tables => $self->_scan_data_dir,
1062                join_summaries   => [],
1063                filter_specs     => $filter_specs,
1064                filters_json     => $filters_json,
1065                export_url       => $self->_build_export_url($lspec, [], $filter_specs),
1066        );
1067}
1068
1069 - 1104
=head2 import_url

C<GET /import> -- Fetch a remote HTML page, extract the first (or selected)
table, and render it as a sortable data grid.

=head3 API SPECIFICATION

=head4 INPUT

  ?url=          string   Required.  Full http:// or https:// URL of the page
                          that contains the HTML table.  Returns home with an
                          error message if the scheme is wrong, the fetch fails,
                          or no table is found at the specified index.
  ?table_index=  integer  Zero-based index of the C<< <table> >> to use when
                          the page contains more than one table (default: 0).
  ?f=            string   (repeatable) Filter spec: "col:op:val".

=head4 OUTPUT

On success renders C<dashboard.html.tt> with the same stash shape as C<open_file>,
plus C<source_url> (the original URL) for C<localStorage> tracking.

On error re-renders C<home.html.tt> with an C<error> stash variable.

=head3 MESSAGES

  error_url_required  -- empty or missing C<url> param
  error_url_invalid   -- URL does not begin with http:// or https://
  error_url_fetch     -- LWP fetch failure or no table found at the index

=head3 EXAMPLE

  GET /import?url=https://matrix.perl-magpie.org/dist/Database-Abstraction/
  GET /import?url=https://example.com/data.html&table_index=2

=cut
1105
1106
28
28
28
276250
34
20
sub import_url ($self) {
1107
28
54
        my ($platform, $language) = $self->_resolve_template;
1108
1109
28
52
        my $url = $self->param('url') // '';
1110
28
3434
        my $idx = $self->param('table_index') // 0;
1111
28
663
        $idx    = 0 unless $idx =~ /\A[0-9]+\z/;
1112
1113
25
25
25
25
30
23
25
22
        my $err_home = sub ($key, @args) {
1114
25
55
                $self->render(
1115                        template => "$platform/$language/home",
1116                        handler  => 'tt',
1117                        format   => 'html',
1118                        tables   => $self->_scan_data_dir,
1119                        title    => 'Choose a Database',
1120                        error    => $self->_i18n($key, @args),
1121                );
1122
28
60
        };
1123
1124
28
49
        return $err_home->('error_url_required') unless length $url;
1125
23
113
        return $err_home->('error_url_invalid', $url)
1126                unless $url =~ m{\Ahttps?://}i;
1127
17
36
        return $err_home->('error_url_ssrf', $url)
1128                unless _is_safe_url($url);
1129
1130
6
6
8
14
        my $source = eval { $self->open_table('', url => $url, html_table_index => $idx) };
1131
6
401
        return $err_home->('error_url_fetch', $url, $@ // 'unknown error') if $@ || !$source;
1132
1133
5
5
5
10
        my $records = eval { $source->fetch_all };
1134
5
432
        return $err_home->('error_url_fetch', $url, $@ // 'empty result') if $@;
1135
1136
3
8
        my $label   = $source->table_name;
1137
3
4
        my $lspec   = "url:$url";
1138
3
8
        my @columns = _get_columns($source, $records);
1139
3
7
        my ($filtered, $filter_specs, $filters_json) = $self->_apply_filters($records);
1140
1141
3
10
        $self->render(
1142                template         => "$platform/$language/dashboard",
1143                handler          => 'tt',
1144                format           => 'html',
1145                records          => $filtered,
1146                columns          => \@columns,
1147                table            => $label,
1148                title            => $label,
1149                source_url       => $url,
1150                back_url         => '/',
1151                back_label       => 'Choose another database',
1152                left_spec        => $lspec,
1153                combine_specs    => [],
1154                current_joins    => [],
1155                available_tables => $self->_scan_data_dir,
1156                join_summaries   => [],
1157                filter_specs     => $filter_specs,
1158                filters_json     => $filters_json,
1159                export_url       => $self->_build_export_url($lspec, [], $filter_specs),
1160        );
1161}
1162
1163 - 1200
=head2 columns_api

C<GET /api/columns> -- Return column names for a table or file as JSON.

Used by the join UI to populate the right-key dropdown without a page reload.

=head3 API SPECIFICATION

=head4 INPUT

  ?table=  string   Table name from C<data_dir>.
  ?path=   string   Absolute path to a data file.

One of C<table> or C<path> must be present; if both are, C<table> takes
precedence.

=head4 OUTPUT

  200 application/json   { "columns": ["col1", "col2", ...] }
  404 application/json   { "error": "not found" }

=head3 MESSAGES

None produced by this action directly; internal errors yield a 404.

=head3 FORMAL SPECIFICATION

  columns_api == lambda self .
    let src = open_spec(table_param or path_param) in
    pre  src /= undef
    post render_json({ columns: get_columns(src) })

=head3 EXAMPLE

  GET /api/columns?table=sales  -> {"columns":["product","region","amount"]}
  GET /api/columns?table=noexist -> 404

=cut
1201
1202
14
14
14
99509
12
14
sub columns_api ($self) {
1203
14
33
        my $table_name = $self->param('table');
1204
14
1680
        my $path       = $self->param('path');
1205
1206
14
305
        my $source;
1207
14
87
        if (defined $table_name && $table_name =~ $TABLE_NAME_RE) {
1208
9
68
                my $data_dir = $self->app->home->child($self->app->config->{data_dir} // 'data');
1209                return $self->render(json => { error => 'not found' }, status => 404)
1210
9
45
189
577
                        unless grep { -f $data_dir->child(lc($table_name) . ".$_") } @SUPPORTED_EXT;
1211
7
7
89
21
                $source = eval { $self->open_table(lc $table_name, directory => $data_dir->to_string) };
1212        }
1213        elsif (defined $path) {
1214
3
3
5
6
                my $file = eval { Mojo::File->new($path)->realpath };
1215
3
90
                if (defined $file && -f $file && $file->basename =~ $EXT_RE) {
1216
2
88
                        my $dir = $file->dirname->to_string;
1217
2
53
                        (my $tbl = $file->basename) =~ s/\.[^.]+\z//;
1218
2
2
26
6
                        $source = eval { $self->open_table(lc $tbl, directory => $dir) };
1219                }
1220        }
1221
1222
12
70
        return $self->render(json => { error => 'not found' }, status => 404) unless $source;
1223
1224        # Use _get_columns to ensure consistent ordering with the view action.
1225
9
1
20
1
        my $recs = ($source->columns ? [] : eval { $source->fetch_all } // []);
1226
9
21
        my @cols = _get_columns($source, $recs);
1227
1228
9
29
        $self->render(json => { columns => \@cols });
1229}
1230
1231 - 1280
=head2 join_tables

C<GET /join> -- Perform one or more left joins and render the merged table.

=head3 API SPECIFICATION

=head4 INPUT

  ?l=    string   Required. Left table spec: "table:name" or "path:/abs/path".
                  Returns 404 if unresolvable.
  ?j=    string   (repeatable) Join step: "<right-spec>|<left-key>|<right-key>".
                  Invalid or non-existent steps are silently skipped.
  ?f=    string   (repeatable) Result filter: "col:op:val".

=head4 OUTPUT

Renders C<dashboard.html.tt> with the merged/filtered result set.

=head3 MESSAGES

  error_table_open   -- Left table fetch threw (re-renders home with error).

=head3 FORMAL SPECIFICATION

  join_tables == lambda self .
    let left = open_spec(param('l')) in
    pre  left /= undef
    let joined  = fold(_left_join, left, param_list('j')) in
    let filtered = fold(_apply_filter_spec, joined, param_list('f')) in
    post render(dashboard, records: filtered)

=head3 PSEUDOCODE

  1. Parse and open left table spec; 404 on failure.
  2. For each j= param (in order):
       a. Parse right-spec, left-key, right-key from "|"-split.
       b. Verify left-key exists in current column list.
       c. Open right table; skip step on any error.
       d. Verify right-key exists in right column list.
       e. Call _left_join; update records and columns.
  3. Apply all f= filters via _apply_filter_spec.
  4. Build stable table key for localStorage ("join:left:right1:...").
  5. Render dashboard.

=head3 EXAMPLE

  GET /join?l=table:sales&j=table:products|product|name
    -> merged left join of sales and products on product/name columns

=cut
1281
1282
22
22
22
187095
23
21
sub join_tables ($self) {
1283
22
51
        my ($platform, $language) = $self->_resolve_template;
1284
1285
22
49
        my $left_spec = $self->param('l') // '';
1286
22
3102
        my ($left_src, $left_label) = $self->_open_spec($left_spec);
1287
22
125
        return $self->reply->not_found unless $left_src;
1288
1289
15
15
14
34
        my $left_recs = eval { $left_src->fetch_all };
1290
15
29
        if ($@) {
1291
0
0
                return $self->render(
1292                        template => "$platform/$language/home",
1293                        handler  => 'tt',
1294                        format   => 'html',
1295                        tables   => [],
1296                        title    => 'Error',
1297                        error    => $self->_i18n('error_table_open', $left_label, $@),
1298                );
1299        }
1300
15
34
        $left_recs //= [];
1301
1302
15
34
        my @left_cols = _get_columns($left_src, $left_recs);
1303
15
15
21
30
        my @join_specs = @{ $self->every_param('j') };
1304
1305
15
527
        my ($records, @columns, @summaries) = ($left_recs, @left_cols);
1306
1307
15
26
        for my $jspec (@join_specs) {
1308
10
35
                my ($right_spec, $left_key, $right_key) = split /\|/, $jspec, 3;
1309
10
45
                next unless defined $right_spec && defined $left_key && defined $right_key;
1310
1311                # O(1) hash-set probe instead of O(C) grep for each join step.
1312
10
53
13
54
                my %col_set = map { $_ => 1 } @columns;
1313
10
22
                next unless $col_set{$left_key};
1314
1315
10
21
                my ($right_src, $right_label) = $self->_open_spec($right_spec);
1316
10
86
                next unless $right_src;
1317
1318
8
8
7
15
                my $right_recs = eval { $right_src->fetch_all } // [];
1319
8
17
                next if $@;
1320
8
12
                my @right_cols = _get_columns($right_src, $right_recs);
1321
8
29
12
29
                my %right_set  = map { $_ => 1 } @right_cols;
1322
8
22
                next unless $right_set{$right_key};
1323
1324
8
24
                ($records, my $new_cols) = _left_join(
1325                        $records, \@columns, $left_key,
1326                        $right_recs, \@right_cols, $right_key, $right_label,
1327                );
1328
8
16
                @columns = @$new_cols;
1329
8
45
                push @summaries, { label => $right_label, left_key => $left_key, right_key => $right_key };
1330        }
1331
1332
15
473
        my ($filtered, $filter_specs, $filters_json) = $self->_apply_filters($records);
1333
1334
15
21
        my $title     = $left_label;
1335
15
8
28
17
        $title       .= ' + ' . join(' + ', map { $_->{label} } @summaries) if @summaries;
1336
15
26
        my $table_key = 'join:' . lc($left_label);
1337
15
33
        $table_key   .= ':' . lc($_->{label}) for @summaries;
1338
1339
15
52
        $self->render(
1340                template         => "$platform/$language/dashboard",
1341                handler          => 'tt',
1342                format           => 'html',
1343                records          => $filtered,
1344                columns          => \@columns,
1345                table            => $table_key,
1346                title            => $title,
1347                back_url         => $self->_spec_to_url($left_spec),
1348                back_label       => "Back to $left_label",
1349                left_spec        => $left_spec,
1350                combine_specs    => [],
1351                current_joins    => \@join_specs,
1352                available_tables => $self->_scan_data_dir,
1353                join_summaries   => \@summaries,
1354                filter_specs     => $filter_specs,
1355                filters_json     => $filters_json,
1356                export_url       => $self->_build_export_url($left_spec, \@join_specs, $filter_specs),
1357        );
1358}
1359
1360 - 1409
=head2 combine_tables

C<GET /combine> -- Stack rows from two or more tables vertically into a single
unified view.

Unlike C</join> (which appends columns from a matching row in a second table),
C</combine> appends the I<rows> from a second table beneath the rows of the
first.  All columns from all sources appear as headers; where a row has no
value for a particular column (because that column did not exist in its source
file) the cell is left blank.

This is equivalent to a SQL C<UNION ALL> across heterogeneous schemas.

=head3 API SPECIFICATION

=head4 INPUT

  ?l=    string   (required) Left table spec: "table:name" or "path:/abs/path".
                  Returns 404 if unresolvable.
  ?c=    string   (repeatable) Additional table spec to stack beneath the
                  current result.  Invalid or non-existent specs are silently
                  skipped.
  ?f=    string   (repeatable) Result filter: "col:op:val".

=head4 OUTPUT

Renders C<dashboard.html.tt> with the combined result set.

=head3 MESSAGES

  error_table_open   -- Left table fetch threw (re-renders home with error).

=head3 FORMAL SPECIFICATION

  combine_tables == lambda self .
    let left = open_spec(param('l')) in
    pre  left /= undef
    let sources  = [left] ++ map(open_spec, param_list('c')) in
    let combined = _combine_tables(sources) in
    let filtered = fold(_apply_filter_spec, combined, param_list('f')) in
    post render(dashboard, records: filtered)

=head3 EXAMPLE

  GET /combine?l=table:cats&c=table:dogs
    -> unified view: all cat and dog rows; Species/Name/Color/Breed/Eye Color
       appear for both; Environment (cats-only) and Sex/Fixed (dogs-only)
       are blank where the source file lacks that column.

=cut
1410
1411
3
3
3
19217
3
2
sub combine_tables ($self) {
1412
3
9
        my ($platform, $language) = $self->_resolve_template;
1413
1414
3
6
        my $left_spec = $self->param('l') // '';
1415
3
447
        my ($left_src, $left_label) = $self->_open_spec($left_spec);
1416
3
5
        return $self->reply->not_found unless $left_src;
1417
1418
3
3
2
6
        my $left_recs = eval { $left_src->fetch_all };
1419
3
6
        if ($@) {
1420
0
0
                return $self->render(
1421                        template => "$platform/$language/home",
1422                        handler  => 'tt',
1423                        format   => 'html',
1424                        tables   => [],
1425                        title    => 'Error',
1426                        error    => $self->_i18n('error_table_open', $left_label, $@),
1427                );
1428        }
1429
3
5
        $left_recs //= [];
1430
1431
3
5
        my @left_cols     = _get_columns($left_src, $left_recs);
1432
3
3
3
5
        my @combine_specs = @{ $self->every_param('c') };
1433
1434
3
120
        my @sources   = ([$left_recs, \@left_cols]);
1435
3
3
        my @summaries;
1436
1437
3
57
        for my $cspec (@combine_specs) {
1438
3
5
                my ($csrc, $clabel) = $self->_open_spec($cspec);
1439
3
5
                next unless $csrc;
1440
3
3
2
4
                my $crecs = eval { $csrc->fetch_all } // [];
1441
3
4
                next if $@;
1442
3
5
                my @ccols = _get_columns($csrc, $crecs);
1443
3
3
                push @sources,   [$crecs, \@ccols];
1444
3
11
                push @summaries, { label => $clabel };
1445        }
1446
1447
3
147
        my ($records, $cols_ref) = @sources > 1
1448                ? _combine_tables(\@sources)
1449                : ($left_recs, \@left_cols);
1450
3
5
        my @columns = @$cols_ref;
1451
1452
3
9
        my ($filtered, $filter_specs, $filters_json) = $self->_apply_filters($records);
1453
1454
3
3
        my $title     = $left_label;
1455
3
3
5
19
        $title       .= ' + ' . join(' + ', map { $_->{label} } @summaries) if @summaries;
1456
3
5
        my $table_key = 'combine:' . lc($left_label);
1457
3
7
        $table_key   .= ':' . lc($_->{label}) for @summaries;
1458
1459
3
8
        $self->render(
1460                template         => "$platform/$language/dashboard",
1461                handler          => 'tt',
1462                format           => 'html',
1463                records          => $filtered,
1464                columns          => \@columns,
1465                table            => $table_key,
1466                title            => $title,
1467                back_url         => $self->_spec_to_url($left_spec),
1468                back_label       => "Back to $left_label",
1469                left_spec        => $left_spec,
1470                combine_specs    => \@combine_specs,
1471                current_joins    => [],
1472                available_tables => $self->_scan_data_dir,
1473                join_summaries   => \@summaries,
1474                filter_specs     => $filter_specs,
1475                filters_json     => $filters_json,
1476                export_url       => $self->_build_export_url($left_spec, [], $filter_specs, \@combine_specs),
1477        );
1478}
1479
1480 - 1534
=head2 export_data

C<GET /export> -- Stream the current logical view as a browser file download.

=head3 API SPECIFICATION

=head4 INPUT

  ?l=        string   (required) Left table spec.
  ?j=        string   (repeatable) Join steps.
  ?f=        string   (repeatable) Filter specs.
  ?format=   string   "csv" (default) or "sqlite".

=head4 DOMAIN CONSTRAINTS: ?format=

The comparison is C<$format eq 'sqlite'> -- case-sensitive, exact match.

=over 4

=item Valid partitions

C<csv> (explicit CSV), C<sqlite> (exact lowercase, SQLite binary),
absent/undef (defaults to CSV).

=item Invalid partitions (all fall back to CSV)

C<SQLITE> (uppercase, not equal to C<'sqlite'>), C<Sqlite> (mixed
case), C<sqlit> (truncated), C<sqlite1> (extra character), C<json>
(unknown format).

=back

=head4 OUTPUT

  200 text/csv                  or application/vnd.sqlite3
  404 application/json          { "error": "Table not found" }

=head3 MESSAGES

None beyond the 404 response.

=head3 FORMAL SPECIFICATION

  export_data == lambda self .
    let (recs, cols, label) = _run_export_pipeline() in
    pre  recs /= undef
    post if format = 'sqlite' then render_sqlite(recs, cols)
         else render_csv(recs, cols)

=head3 EXAMPLE

  GET /export?l=table:sales&format=csv     -> downloads sales.csv
  GET /export?l=table:sales&format=sqlite  -> downloads sales.db

=cut
1535
1536
39
39
39
272186
43
32
sub export_data ($self) {
1537
39
80
        my ($records, $columns, $left_label) = $self->_run_export_pipeline;
1538
39
1780
        return $self->reply->not_found unless $records;
1539
1540
33
73
        my $format = $self->param('format') // 'csv';
1541
33
826
        $format    = 'csv' unless $format eq 'sqlite';
1542
33
79
        (my $safe_name = lc $left_label) =~ s/[^a-z0-9_]+/_/g;
1543
1544
33
99
        return $format eq 'sqlite'
1545                ? $self->_render_sqlite($records, $columns, $safe_name)
1546                : $self->_render_csv($records, $columns, $safe_name);
1547}
1548
1549 - 1616
=head2 export_write

C<POST /export> -- Write the current logical view to a chosen filesystem path.

=head3 API SPECIFICATION

=head4 INPUT

  l=         string   (required) Left table spec.
  j=         string   (repeatable) Join steps.
  f=         string   (repeatable) Filter specs.
  dir=       string   Target directory (must exist; resolved via realpath).
  filename=  string   Output filename including extension.  Extension determines
                      format: C<.csv> -> RFC 4180 CSV; C<.sql> -> SQLite.

=head4 DOMAIN CONSTRAINTS: filename=

The extension check uses C</\.csv\z/i> (CSV) or C</\.sql\z/i> (SQLite):
the C</i> flag makes matching case-insensitive, so C<.CSV> and C<.SQL>
are accepted alongside lowercase forms.  Everything else returns 415.

Path separator characters (C</> and C<\>) in C<filename> are stripped
first via C<m{([^/\\]+)\z}> -- only the basename is kept, preventing
directory traversal.

=over 4

=item Valid partitions

C<report.csv>, C<report.sql>, C<REPORT.CSV>, C<report.SQL>.  A
single-character stem (C<a.csv>) is also valid.

=item Invalid partitions (415)

C<report.txt>, C<report.json>, C<report> (no extension), empty string.

=back

=head4 OUTPUT

  200 application/json   { "saved": "/abs/path/to/file" }
  404 application/json   { "error": "..." }  -- bad dir or table not found
  415 application/json   { "error": "..." }  -- unsupported extension
  500 application/json   { "error": "..." }  -- write failure

=head3 MESSAGES

  error_dir_not_found   -- realpath of dir failed or result is not a directory.
  error_ext_required    -- filename extension is not .csv or .sql.
  error_write_failed    -- DBI or filesystem write threw.

=head3 FORMAL SPECIFICATION

  export_write == lambda self .
    let dir  = realpath(param('dir'))      in
    let file = dir / strip_path(param('filename')) in
    pre  is_dir(dir) /\ ext(file) in {csv, sql}
    let (recs, cols, _) = _run_export_pipeline() in
    pre  recs /= undef
    post spurt(file, serialise(recs, cols))
         /\ render_json({ saved: file.to_string })

=head3 EXAMPLE

  POST /export  l=table:sales dir=/home/user/exports filename=report.csv
    -> { "saved": "/home/user/exports/report.csv" }

=cut
1617
1618
38
38
38
207989
41
26
sub export_write ($self) {
1619
38
68
        my $dir      = $self->param('dir')      // '';
1620
38
6562
        my $filename = $self->param('filename') // '';
1621
1622
38
38
904
100
        my $dest_dir = eval { Mojo::File->new($dir)->realpath };
1623
38
1942
        unless (defined $dest_dir && -d $dest_dir) {
1624
5
21
                return $self->render(
1625                        json   => { error => $self->_i18n('error_dir_not_found', $dir) },
1626                        status => 404,
1627                );
1628        }
1629
1630        # Strip any path separators the browser might include.
1631
33
257
        ($filename) = $filename =~ m{([^/\\]+)\z};
1632
33
30
        my $format;
1633
33
19
169
25
        if    ($filename && $filename =~ /\.csv\z/i) { $format = 'csv';    }
1634
4
5
        elsif ($filename && $filename =~ /\.sql\z/i) { $format = 'sqlite'; }
1635        else {
1636
10
22
                return $self->render(
1637                        json   => { error => $self->_i18n('error_ext_required') },
1638                        status => 415,
1639                );
1640        }
1641
1642
23
45
        my ($records, $columns, $left_label) = $self->_run_export_pipeline;
1643
23
1129
        return $self->render(json => { error => 'Table not found' }, status => 404)
1644                unless $records;
1645
1646
21
52
        my $dest = $dest_dir->child($filename);
1647
21
211
        eval {
1648
21
38
                if ($format eq 'csv') {
1649
17
30
                        $dest->spurt(_serialize_csv($records, $columns));
1650                }
1651                else {
1652
4
10
                        $dest->spurt($self->_write_sqlite_db($records, $columns));
1653                }
1654        };
1655
21
1774
        return $self->render(
1656                json   => { error => $self->_i18n('error_write_failed', $@) },
1657                status => 500,
1658        ) if $@;
1659
1660
21
44
        $self->render(json => { saved => $dest->to_string });
1661}
1662
1663 - 1701
=head2 dirs_api

C<GET /api/dirs> -- Return a JSON directory listing for the export panel.

=head3 API SPECIFICATION

=head4 INPUT

  ?path=   string   Directory to list (default: $HOME).  Returns 404 when not
                    a directory.  Hidden entries (names starting with ".") are
                    excluded.

=head4 OUTPUT

  200 application/json
    {
      "path":   "/abs/path",
      "parent": "/abs/parent" or null (at filesystem root),
      "dirs":   [ { "name": "alpha", "path": "/abs/path/alpha" }, ... ]
    }

  404 application/json   { "error": "Not a directory" }

=head3 MESSAGES

None beyond the 404 response.

=head3 FORMAL SPECIFICATION

  dirs_api == lambda self .
    let dir = realpath(param('path') // HOME) in
    pre  is_dir(dir)
    post render_json({ path: dir, parent: parent(dir), dirs: subdirs(dir) })

=head3 EXAMPLE

  GET /api/dirs?path=/home/user  -> {"path":"/home/user","parent":"/home","dirs":[...]}

=cut
1702
1703
11
11
11
51576
12
6
sub dirs_api ($self) {
1704
11
19
        my $raw = $self->param('path') // $ENV{HOME} // '/';
1705
11
11
1290
32
        my $dir = eval { Mojo::File->new($raw)->realpath };
1706
11
832
        return $self->render(json => { error => 'Not a directory' }, status => 404)
1707                unless defined $dir && -d $dir;
1708
1709
6
46
        my $listing = $self->_list_dir($dir, 0);
1710
6
15
        my $parent  = $dir->dirname;
1711
1712        $self->render(json => {
1713                path   => $dir->to_string,
1714                parent => ($parent->to_string ne $dir->to_string ? $parent->to_string : undef),
1715                dirs   => $listing->{dirs},
1716
6
170
        });
1717}
1718
1719 - 1768
=head2 stat_api

C<GET /api/stat> -- Return filesystem metadata for a file path.

Used by the home page tooltip to show modification time and size for recently
opened files without a page reload.

=head3 API SPECIFICATION

=head4 INPUT

  ?path=   string   Absolute path to query.  Returns HTTP 400 when absent.

=head4 OUTPUT

  200 application/json   (file exists)
    { "exists": true, "path": "/resolved/path", "mtime": 1700000000, "size": 4096 }

  200 application/json   (file does not exist)
    { "exists": false, "path": "/original/path" }

  400 application/json   { "error": "\"path\" parameter is required" }

C<mtime> is Unix epoch seconds.  C<size> is bytes.  A missing/unresolvable
path returns HTTP 200 with C<exists: false> (not 404) so the UI can
distinguish "file was deleted" from a request error.

=head3 MESSAGES

  error_path_required   -- the "path" query parameter was not supplied.

=head3 FORMAL SPECIFICATION

  stat_api == lambda self .
    let path = param('path') in
    pre  path /= undef
    let file = realpath(path) in
    post if is_file(file) then
           render_json({ exists: true, mtime: mtime(file), size: size(file) })
         else render_json({ exists: false })

=head3 EXAMPLE

  GET /api/stat?path=/data/sales.csv
    -> { "exists": true, "path": "/data/sales.csv", "mtime": 1700000000, "size": 1234 }

  GET /api/stat?path=/deleted.csv
    -> { "exists": false, "path": "/deleted.csv" }

=cut
1769
1770
26
26
26
133921
30
23
sub stat_api ($self) {
1771
26
49
        my $path = $self->param('path');
1772
26
3036
        unless (defined $path && length $path) {
1773
6
19
                return $self->render(
1774                        json   => { error => $self->_i18n('error_path_required') },
1775                        status => 400,
1776                );
1777        }
1778
1779
20
20
18
39
        my $file = eval { Mojo::File->new($path)->realpath };
1780        # Restrict to files with a supported data extension so stat_api cannot be
1781        # used as a filesystem oracle to probe /etc/shadow, /root/.ssh/, etc.
1782
20
1315
        return $self->render(json => { exists => \0, path => $path })
1783                unless defined $file && -f $file && $file->basename =~ $EXT_RE;
1784
1785
11
418
        my @s = stat $file->to_string;
1786
11
88
        $self->render(json => {
1787                exists => \1,
1788                path   => $file->to_string,
1789                mtime  => $s[9],
1790                size   => $s[7],
1791        });
1792}
1793
1794 - 1835
=head2 upload_file

C<POST /upload> -- Accept a drag-and-dropped data file and return a redirect URL.

The file is saved under its original filename in a managed subdirectory of
C<< <app_home>/.uploads/ >>.  The subdirectory name is randomised so that
concurrent uploads of files with the same name do not collide.

=head3 API SPECIFICATION

=head4 INPUT

Multipart form upload, field name: C<file>.

  file   upload   Supported extensions: csv, db, sql, xml, psv.

=head4 OUTPUT

  200 application/json   { "url": "/open?path=/abs/path/file.csv", "path": "/abs/path/file.csv" }
  400 application/json   { "error": "No file received" }
  415 application/json   { "error": "Unsupported file type. Accepted: ..." }

=head3 MESSAGES

  error_upload_none   -- no file was received in the multipart upload.
  error_upload_ext    -- the file's extension is not in the supported list.

=head3 FORMAL SPECIFICATION

  upload_file == lambda self .
    let upload = req.upload('file') in
    pre  upload /= undef /\ basename(upload.filename) =~ EXT_RE
    let dest = home/.uploads/<random>/<filename> in
    post upload.move_to(dest)
         /\ render_json({ url: '/open?path=' ++ url_escape(dest), path: dest })

=head3 EXAMPLE

  POST /upload  (multipart: file=@sales.csv)
    -> { "url": "/open?path=%2F...%2Fsales.csv", "path": "/.../.uploads/abc123/sales.csv" }

=cut
1836
1837
29
29
29
201290
32
21
sub upload_file ($self) {
1838
29
42
        my $upload = $self->req->upload('file');
1839
29
2273
        unless ($upload && $upload->filename) {
1840
4
10
                return $self->render(
1841                        json   => { error => $self->_i18n('error_upload_none') },
1842                        status => 400,
1843                );
1844        }
1845
1846        # Enforce upload size limit.  max_request_size in startup() sets the
1847        # transport-layer cap, but Mojolicious still calls the controller when the
1848        # limit fires (with req->is_limit_exceeded true and partial content in the
1849        # upload asset).  Check is_limit_exceeded first; fall through to an asset
1850        # size check as a belt-and-braces guard for any bytes that slipped through.
1851
25
97
        if ($self->req->is_limit_exceeded || $upload->size > $MAX_UPLOAD_BYTES) {
1852
3
23
                return $self->render(
1853                        json   => { error => $self->_i18n('error_upload_too_large', $MAX_UPLOAD_MIB) },
1854                        status => 413,
1855                );
1856        }
1857
1858        # /s so .* matches \n; a percent-decoded newline inside a browser filename
1859        # must not silently truncate the directory-strip, leaving "dir\ncomponent" in.
1860
22
345
        (my $filename = $upload->filename) =~ s{.*[/\\]}{}s;
1861
22
126
        unless ($filename && $filename =~ $EXT_RE) {
1862
5
41
                return $self->render(
1863                        json   => { error => $self->_i18n('error_upload_ext') },
1864                        status => 415,
1865                );
1866        }
1867
1868        # Store uploads in <app_home>/.uploads/<random_subdir>/<original_filename>.
1869        # Using the original filename is essential: Database::Abstraction derives
1870        # the table name from the file stem, so "sales.csv" must stay "sales.csv".
1871        # The random subdirectory prevents collisions from concurrent same-name uploads.
1872
17
157
        my $uploads_base = $self->app->home->child('.uploads');
1873
17
279
        $uploads_base->make_path unless -d $uploads_base;
1874
17
344
        my $sub_dir = tempdir(DIR => $uploads_base->to_string, CLEANUP => 0);
1875
17
3389
        my $dest    = Mojo::File->new($sub_dir)->child($filename)->to_string;
1876
17
272
        $upload->move_to($dest);
1877
1878
17
1208
        $self->render(json => {
1879                url  => '/open?path=' . url_escape($dest),
1880                path => $dest,
1881        });
1882}
1883
18841;
1885