| File: | blib/lib/Database/BI/Controller/Dashboard.pm |
| Coverage: | 92.4% |
| line | stmt | bran | cond | sub | time | code |
|---|---|---|---|---|---|---|
| 1 | package Database::BI::Controller::Dashboard; | |||||
| 2 | ||||||
| 3 | our $VERSION = '0.002.0'; | |||||
| 4 | ||||||
| 5 | 12 12 12 | 10641 10 39 | use Mojo::Base 'Mojolicious::Controller', -strict, -signatures; | |||
| 6 | ||||||
| 7 | 12 12 12 | 7041 44 357 | use Carp qw(croak carp); | |||
| 8 | 12 12 12 | 41 25 193 | use Mojo::File; | |||
| 9 | 12 12 12 | 24 15 207 | use Mojo::JSON qw(encode_json); | |||
| 10 | 12 12 12 | 15 9 202 | use Mojo::Util qw(url_escape encode); | |||
| 11 | 12 12 12 | 24 8 275 | use File::Temp qw(tempfile tempdir); | |||
| 12 | 12 12 12 | 22 11 205 | use Readonly; | |||
| 13 | 12 12 12 | 21 17 247 | use Socket qw(inet_aton); | |||
| 14 | 12 12 12 | 17 14 53 | use Sub::Protected; | |||
| 15 | ||||||
| 16 | # --------------------------------------------------------------------------- | |||||
| 17 | # Constants | |||||
| 18 | # --------------------------------------------------------------------------- | |||||
| 19 | ||||||
| 20 | # File extensions that Database::Abstraction can probe, in probe order. | |||||
| 21 | # Note: D::A uses ".sql" for SQLite -- NOT ".sqlite". | |||||
| 22 | Readonly my @SUPPORTED_EXT => qw( csv db sql xml psv ); | |||||
| 23 | # Use \z (absolute end-of-string) not $ (which permits a trailing \n before \z). | |||||
| 24 | # A query param decoded from "file.csv%0A" has basename "sales.csv\n"; without \z | |||||
| 25 | # that passes the extension guard and reaches realpath with an embedded newline. | |||||
| 26 | Readonly my $EXT_RE => do { my $p = join '|', @SUPPORTED_EXT; qr/\.(?:$p)\z/i }; | |||||
| 27 | ||||||
| 28 | # Table name safe-identifier pattern -- must match DataSource's TABLE_NAME_RE | |||||
| 29 | # exactly: first character must be a letter or underscore (not a digit), so | |||||
| 30 | # that every table name the controller accepts can also be opened by DataSource | |||||
| 31 | # without an unhandled croak. | |||||
| 32 | Readonly my $TABLE_NAME_RE => qr/\A[A-Za-z_][A-Za-z0-9_]*\z/; | |||||
| 33 | ||||||
| 34 | # URL spec pattern shared by _open_spec and _spec_to_url. | |||||
| 35 | # qr{} delimiter avoids the \/ escaping noise required by the // form. | |||||
| 36 | # /s: makes . match \n so a percent-decoded newline inside a URL does not | |||||
| 37 | # silently truncate the captured URL before the \z end-of-string anchor. | |||||
| 38 | Readonly my $URL_SPEC_RE => qr{\Aurl:(https?://.+)\z}si; | |||||
| 39 | ||||||
| 40 | # --------------------------------------------------------------------------- | |||||
| 41 | # I18N message dictionary for all user-visible strings in this controller. | |||||
| 42 | # To plug in a Locale::Maketext backend, extend _i18n() below. | |||||
| 43 | # --------------------------------------------------------------------------- | |||||
| 44 | ||||||
| 45 | Readonly my %MESSAGES => ( | |||||
| 46 | error_table_invalid => 'Invalid table name "%s"', | |||||
| 47 | error_table_open => 'Could not open table "%s": %s', | |||||
| 48 | error_file_open => 'Could not open "%s": %s', | |||||
| 49 | error_no_path => 'No path specified', | |||||
| 50 | error_not_found => 'File or directory not found: %s', | |||||
| 51 | error_dir_not_found => 'Directory not found: %s', | |||||
| 52 | error_write_failed => 'Write failed: %s', | |||||
| 53 | error_ext_required => 'Use a .csv or .sql filename extension', | |||||
| 54 | error_upload_none => 'No file received', | |||||
| 55 | error_upload_ext => 'Unsupported file type. Accepted: CSV, PSV, XML, SQLite (.sql)', | |||||
| 56 | error_upload_too_large => 'File too large (maximum %s MiB)', | |||||
| 57 | error_path_required => '"path" parameter is required', | |||||
| 58 | error_url_required => 'Please enter a URL', | |||||
| 59 | error_url_invalid => '"%s" is not a valid http:// or https:// URL', | |||||
| 60 | error_url_fetch => 'Could not load HTML table from "%s": %s', | |||||
| 61 | error_url_ssrf => '"%s" resolves to a private or reserved address and cannot be fetched', | |||||
| 62 | ); | |||||
| 63 | ||||||
| 64 | # Maximum accepted upload body size. Enforced both here (application layer) | |||||
| 65 | # and via Mojolicious max_request_size (transport layer) set in startup(). | |||||
| 66 | # Must match $MAX_REQUEST_SIZE in BI.pm. | |||||
| 67 | Readonly my $MAX_UPLOAD_MIB => 50; | |||||
| 68 | Readonly my $MAX_UPLOAD_BYTES => $MAX_UPLOAD_MIB * 1_048_576; | |||||
| 69 | ||||||
| 70 | # --------------------------------------------------------------------------- | |||||
| 71 | # Protected helpers | |||||
| 72 | # --------------------------------------------------------------------------- | |||||
| 73 | ||||||
| 74 | # _i18n($self, $key, @sprintf_args) -> $string | |||||
| 75 | # | |||||
| 76 | # Purpose: Look up a user-visible string from %MESSAGES and apply sprintf | |||||
| 77 | # for positional arguments. The entry point for future i18n | |||||
| 78 | # backend integration (e.g. Locale::Maketext). | |||||
| 79 | # Entry: $key must be a key in %MESSAGES; @args are sprintf positionals. | |||||
| 80 | # Exit: Returns the formatted string, or a fallback containing $key. | |||||
| 81 | 78 78 78 78 78 | 3057 67 83 80 67 | sub _i18n :Protected ($self, $key, @args) { | |||
| 82 | 78 | 265 | my $tmpl = $MESSAGES{$key} // return "Internal error: unknown message key '$key'"; | |||
| 83 | 77 | 567 | return @args ? sprintf($tmpl, @args) : $tmpl; | |||
| 84 | 12 12 12 | 3037 6 218 | } | |||
| 85 | ||||||
| 86 | # _is_safe_url($url) -> bool | |||||
| 87 | # | |||||
| 88 | # Purpose: SSRF guard. Blocks the most dangerous classes of Server-Side Request | |||||
| 89 | # Forgery targets: loopback aliases (localhost, 127/8) and literal | |||||
| 90 | # private/link-local/CGNAT IPv4 addresses in the URL host component. | |||||
| 91 | # | |||||
| 92 | # Design rationale for scope: | |||||
| 93 | # Resolving hostnames via DNS and checking the result is ineffective: a | |||||
| 94 | # separate DNS lookup happens at LWP connect time (TOCTOU / DNS rebinding | |||||
| 95 | # window). The authoritative defence against hostname-based SSRF is a | |||||
| 96 | # network-layer egress firewall. This function handles the Perl-layer | |||||
| 97 | # interception for the most common patterns that operators cannot easily | |||||
| 98 | # filter at the network level: bare loopback aliases and literal private IPs | |||||
| 99 | # hard-coded by an attacker. | |||||
| 100 | # | |||||
| 101 | # Blocked targets: | |||||
| 102 | # localhost / 127.0.0.0/8 / 0.0.0.0 / ::1 -- loopback aliases | |||||
| 103 | # 10.0.0.0/8 -- RFC 1918 private (literal IP) | |||||
| 104 | # 172.16.0.0/12 -- RFC 1918 private (literal IP) | |||||
| 105 | # 192.168.0.0/16-- RFC 1918 private (literal IP) | |||||
| 106 | # 169.254.0.0/16-- link-local; AWS/GCP/Azure metadata endpoint (literal IP) | |||||
| 107 | # 100.64.0.0/10 -- CGNAT / Tailscale shared space (literal IP) | |||||
| 108 | # | |||||
| 109 | # Hostname-based targets (e.g. http://internal.corp.example.com/) are allowed | |||||
| 110 | # at this layer; block them with egress firewall rules instead. | |||||
| 111 | sub _is_safe_url { | |||||
| 112 | 95 | 28466 | my ($url) = @_; | |||
| 113 | 95 | 279 | return 0 unless $url =~ m{\Ahttps?://([^/:?\[\]#]+)}i; | |||
| 114 | 83 | 112 | my $host = lc $1; | |||
| 115 | ||||||
| 116 | # Block well-known loopback aliases. | |||||
| 117 | 83 | 325 | return 0 if $host eq 'localhost' | |||
| 118 | || $host =~ /\A127\./ | |||||
| 119 | || $host eq '0.0.0.0' | |||||
| 120 | || $host eq '::1'; | |||||
| 121 | ||||||
| 122 | # For literal IPv4 addresses only: check private/link-local/CGNAT ranges. | |||||
| 123 | # Skipping DNS for hostname targets avoids a live network call in tests and | |||||
| 124 | # removes the TOCTOU window that makes DNS-resolved checks illusory anyway. | |||||
| 125 | 62 | 151 | return 1 unless $host =~ /\A\d{1,3}(?:\.\d{1,3}){3}\z/; | |||
| 126 | ||||||
| 127 | 47 | 198 | my $packed = inet_aton($host) or return 1; | |||
| 128 | 46 | 70 | my $n = unpack 'N', $packed; | |||
| 129 | ||||||
| 130 | 46 | 70 | return 0 if ($n & 0xFF000000) == 0x0A000000; # 10/8 | |||
| 131 | 38 | 52 | return 0 if ($n & 0xFFF00000) == 0xAC100000; # 172.16/12 | |||
| 132 | 31 | 47 | return 0 if ($n & 0xFFFF0000) == 0xC0A80000; # 192.168/16 | |||
| 133 | 24 | 35 | return 0 if ($n & 0xFFFF0000) == 0xA9FE0000; # 169.254/16 (link-local / metadata) | |||
| 134 | 18 | 26 | return 0 if ($n & 0xFFC00000) == 0x64400000; # 100.64/10 (CGNAT) | |||
| 135 | 12 | 23 | return 1; | |||
| 136 | } | |||||
| 137 | ||||||
| 138 | # _resolve_template($self) -> ($platform, $language) | |||||
| 139 | # | |||||
| 140 | # Purpose: Read platform and language from config, then resolve the Accept-Language | |||||
| 141 | # header to a language code -- falling back to the config default when | |||||
| 142 | # no template directory exists for the resolved language. | |||||
| 143 | # Exit: Returns ($platform, $language) -- both guaranteed non-empty strings. | |||||
| 144 | 213 213 213 | 178 171 167 | sub _resolve_template :Protected ($self) { | |||
| 145 | 213 | 281 | my $conf = $self->app->config; | |||
| 146 | 213 | 1204 | my $platform = $conf->{platform} // 'web'; | |||
| 147 | 213 | 329 | my $default = $conf->{language} // 'en'; | |||
| 148 | 213 | 361 | my $language = $self->_resolve_language($default); | |||
| 149 | 213 | 289 | return ($platform, $language); | |||
| 150 | 12 12 12 | 3770 10 104 | } | |||
| 151 | ||||||
| 152 | # _resolve_language($self, $default) -> $language_code | |||||
| 153 | # | |||||
| 154 | # Purpose: Extract the first two-letter language code from the Accept-Language | |||||
| 155 | # request header, then validate that a template directory exists for | |||||
| 156 | # that language. Falls back to $default when the header is absent, | |||||
| 157 | # unparseable, or points to a non-existent template directory. | |||||
| 158 | # Entry: $default is a non-empty string (e.g. 'en'). | |||||
| 159 | # Exit: Returns a two-letter ISO 639-1 language code string. | |||||
| 160 | # Side Effects: Filesystem stat for the template directory. | |||||
| 161 | 221 221 221 221 | 11670 185 179 151 | sub _resolve_language :Protected ($self, $default) { | |||
| 162 | 221 | 314 | my $accept = $self->req->headers->accept_language // ''; | |||
| 163 | 221 | 2511 | my ($lang) = $accept =~ / | |||
| 164 | \b | |||||
| 165 | ( [a-z]{2} ) # ISO 639-1 primary language subtag (exactly 2 lowercase letters) | |||||
| 166 | (?: - [A-Z]{2} )? # optional ISO 3166-1 region subtag: hyphen + 2 uppercase letters | |||||
| 167 | \b | |||||
| 168 | /x; | |||||
| 169 | 221 | 476 | $lang //= $default; | |||
| 170 | ||||||
| 171 | # Only use the resolved language if templates actually exist for it; | |||||
| 172 | # prevents a 500 error when a browser sends e.g. Accept-Language: de | |||||
| 173 | # but no templates/web/de/ directory is present. | |||||
| 174 | 221 | 319 | if ($lang ne $default) { | |||
| 175 | 7 | 11 | my $platform = $self->app->config->{platform} // 'web'; | |||
| 176 | 7 | 51 | my $dir = $self->app->home->child("templates/$platform/$lang"); | |||
| 177 | 7 | 162 | $lang = $default unless -d $dir; | |||
| 178 | } | |||||
| 179 | 221 | 341 | return $lang; | |||
| 180 | 12 12 12 | 2626 12 98 | } | |||
| 181 | ||||||
| 182 | # _scan_data_dir($self) -> \@table_list | |||||
| 183 | # | |||||
| 184 | # Purpose: Return an arrayref of { name, file } hashrefs for every supported | |||||
| 185 | # data file in the configured data_dir. Used by the index action and | |||||
| 186 | # by the join panel "available tables" dropdown. | |||||
| 187 | # Entry: None. | |||||
| 188 | # Exit: Returns [] when data_dir does not exist or contains no supported files. | |||||
| 189 | # Side Effects: Filesystem directory read. | |||||
| 190 | # | |||||
| 191 | # Optimisation: single-pass map replaces a two-pass grep+map chain, and calls | |||||
| 192 | # basename() only once per entry (the original chain called it twice: once in | |||||
| 193 | # grep to check the extension and again in map to extract the stem). | |||||
| 194 | # ->to_array avoids the extra flat-list expansion that ->each produced. | |||||
| 195 | 136 136 136 | 257 112 87 | sub _scan_data_dir :Protected ($self) { | |||
| 196 | 136 | 227 | my $dir = $self->app->home->child($self->app->config->{data_dir} // 'data'); | |||
| 197 | 136 | 3078 | return [] unless -d $dir; | |||
| 198 | return $dir->list->map(sub { | |||||
| 199 | 952 | 16688 | my $base = $_->basename; | |||
| 200 | 952 | 11852 | return unless $base =~ $EXT_RE; | |||
| 201 | 952 | 3246 | (my $name = $base) =~ s/\.[^.]+\z//; | |||
| 202 | 952 | 1418 | { name => $name, file => $base } | |||
| 203 | 136 | 1194 | })->to_array; | |||
| 204 | 12 12 12 | 2211 11 87 | } | |||
| 205 | ||||||
| 206 | # _open_spec($self, $spec) -> ($DataSource, $label) or () | |||||
| 207 | # | |||||
| 208 | # Purpose: Parse a "table:name" or "path:/abs/path" spec, verify the underlying | |||||
| 209 | # file exists (Database::Abstraction creates empty objects silently for | |||||
| 210 | # missing files, so we must check ourselves), and return a live | |||||
| 211 | # DataSource object and a display label. | |||||
| 212 | # Entry: $spec is a non-empty string. | |||||
| 213 | # Exit: Returns ($datasource, $label) on success; an empty list on any failure | |||||
| 214 | # (invalid spec format, file missing, DataSource init error). | |||||
| 215 | # Side Effects: Filesystem stat; DataSource construction (reads file header). | |||||
| 216 | 104 104 104 104 | 97 101 92 81 | sub _open_spec :Protected ($self, $spec) { | |||
| 217 | 104 | 318 | if ($spec =~ /\Atable:([A-Za-z0-9_]+)\z/) { | |||
| 218 | 75 | 126 | my $table = lc $1; | |||
| 219 | 75 | 112 | my $data_dir = $self->app->home->child($self->app->config->{data_dir} // 'data'); | |||
| 220 | 75 375 | 1435 4228 | return () unless grep { -f $data_dir->child("$table.$_") } @SUPPORTED_EXT; | |||
| 221 | 70 70 | 850 111 | my $src = eval { $self->open_table($table, directory => $data_dir->to_string) }; | |||
| 222 | 70 | 335 | return ($src, $table) if $src && !$@; | |||
| 223 | } | |||||
| 224 | elsif ($spec =~ /\Apath:(.+)\z/) { | |||||
| 225 | 19 19 | 17 45 | my $file = eval { Mojo::File->new($1)->realpath }; | |||
| 226 | 19 | 455 | if (defined $file && -f $file && $file->basename =~ $EXT_RE) { | |||
| 227 | 16 | 508 | my $dir = $file->dirname->to_string; | |||
| 228 | 16 | 341 | (my $table = $file->basename) =~ s/\.[^.]+\z//; | |||
| 229 | 16 16 | 173 43 | my $src = eval { $self->open_table(lc $table, directory => $dir) }; | |||
| 230 | 16 | 68 | return ($src, $file->basename) if $src && !$@; | |||
| 231 | } | |||||
| 232 | } | |||||
| 233 | elsif ($spec =~ $URL_SPEC_RE) { | |||||
| 234 | 2 | 20 | my $url = $1; | |||
| 235 | 2 | 4 | return () unless _is_safe_url($url); | |||
| 236 | 1 1 | 1 2 | my $src = eval { $self->open_table('', url => $url) }; | |||
| 237 | 1 | 5 | return ($src, $src->table_name) if $src && !$@; | |||
| 238 | } | |||||
| 239 | 11 | 144 | return (); | |||
| 240 | 12 12 12 | 3832 12 88 | } | |||
| 241 | ||||||
| 242 | # _spec_to_url($self, $spec) -> $url_string | |||||
| 243 | # | |||||
| 244 | # Purpose: Convert a table/path spec back to a browseable URL (used for the | |||||
| 245 | # back-link on join and export views). | |||||
| 246 | # Entry: $spec is a "table:name" or "path:/abs" string. | |||||
| 247 | # Exit: Returns a URL string beginning with '/'. | |||||
| 248 | 27 27 27 27 | 10642 25 40 18 | sub _spec_to_url :Protected ($self, $spec) { | |||
| 249 | 27 | 125 | return "/view/$1" if $spec =~ /\Atable:([A-Za-z0-9_]+)\z/; | |||
| 250 | 11 | 36 | return '/open?path=' . url_escape($1) if $spec =~ /\Apath:(.+)\z/; | |||
| 251 | 6 | 21 | return '/import?url=' . url_escape($1) if $spec =~ $URL_SPEC_RE; | |||
| 252 | 3 | 13 | return '/'; | |||
| 253 | 12 12 12 | 1884 9 82 | } | |||
| 254 | ||||||
| 255 | # _get_columns($source, $records) -> @column_names | |||||
| 256 | # | |||||
| 257 | # Purpose: Return an ordered column list from a DataSource object. | |||||
| 258 | # For CSV/PSV the DataSource stores the original file-header order. | |||||
| 259 | # For SQLite/XML where no file-header order is available, the fallback | |||||
| 260 | # is: id_column first (only if it actually appears in the data), then | |||||
| 261 | # the remaining columns sorted alphabetically. | |||||
| 262 | # Entry: $source is a DataSource; $records is an arrayref of hashrefs (may be empty). | |||||
| 263 | # Exit: Returns a flat list of column-name strings (may be empty). | |||||
| 264 | # | |||||
| 265 | # Reduction: columns() is a pure accessor -- cache once to avoid a redundant | |||||
| 266 | # method dispatch on the second reference. | |||||
| 267 | sub _get_columns { | |||||
| 268 | 180 | 10067 | my ($source, $records) = @_; | |||
| 269 | 180 | 339 | my $cols = $source->columns; | |||
| 270 | 180 | 436 | return @$cols if $cols; | |||
| 271 | 26 | 45 | return () unless $records->[0]; | |||
| 272 | 24 66 24 | 20 76 68 | my %all = map { $_ => 1 } keys %{ $records->[0] }; | |||
| 273 | 24 | 53 | my $c = $source->id_column; | |||
| 274 | 24 | 89 | my $id = ($c && $all{$c}) ? $c : (sort keys %all)[0]; | |||
| 275 | 24 | 31 | delete $all{$id}; | |||
| 276 | 24 | 56 | return ($id, sort keys %all); | |||
| 277 | } | |||||
| 278 | ||||||
| 279 | # _apply_filter_spec($records, $spec) -> \@filtered_records | |||||
| 280 | # | |||||
| 281 | # Purpose: Apply one "col:op:val" filter to an arrayref of record hashrefs and | |||||
| 282 | # return a new (possibly shorter) arrayref. The original is not mutated. | |||||
| 283 | # Value may contain colons: "col:op:val:with:colons" -- split limit 3 | |||||
| 284 | # puts everything after the second colon into $val. | |||||
| 285 | # Unknown operators fall through to 1 (all rows pass) so new operators | |||||
| 286 | # added in future do not break existing callers. | |||||
| 287 | # Entry: $records is an arrayref of hashrefs; $spec is a colon-delimited string. | |||||
| 288 | # Exit: Returns filtered arrayref (same reference if spec is invalid/unparseable). | |||||
| 289 | # | |||||
| 290 | # Domain Constraints (operator names): | |||||
| 291 | # Operator names are CASE-SENSITIVE. Only the exact lowercase forms below are | |||||
| 292 | # recognised; an uppercase or mixed-case name (e.g. "EQ", "Lt") falls through | |||||
| 293 | # to the default `1` branch and all rows pass unfiltered. | |||||
| 294 | # Valid operators: | |||||
| 295 | # String (value comparison is case-insensitive via lc()): | |||||
| 296 | # eq, ne, contains, starts | |||||
| 297 | # Numeric (Perl <, <=, >, >= coercion): | |||||
| 298 | # lt (strict), le (inclusive), gt (strict), ge (inclusive) | |||||
| 299 | # Unary (val is ignored): | |||||
| 300 | # empty (cell eq ''), notempty (cell ne '') | |||||
| 301 | # Boundary rules for numeric operators: | |||||
| 302 | # le/ge include the boundary value; lt/gt exclude it. | |||||
| 303 | # contains/starts with empty val ('') match every record (index always 0). | |||||
| 304 | # | |||||
| 305 | # Reduction: split(/:/, $spec, 3) always produces at least one defined element, | |||||
| 306 | # so "defined $col" is structurally guaranteed true and is removed. | |||||
| 307 | # "$_->{$col} // ''" replaces the equivalent ternary form. | |||||
| 308 | # | |||||
| 309 | # Optimisation: $lval = lc($val) is computed once before the grep. Without this | |||||
| 310 | # precompute, every case-insensitive operator (eq, ne, contains, starts) would | |||||
| 311 | # call lc($val) O(N) times for N records -- a pure O(N) waste since $val is | |||||
| 312 | # constant within a single filter application. | |||||
| 313 | sub _apply_filter_spec { | |||||
| 314 | 277 | 74754 | my ($records, $spec) = @_; | |||
| 315 | 277 | 362 | my ($col, $op, $val) = split /:/, $spec, 3; | |||
| 316 | 277 | 669 | return $records unless length($col // '') && defined $op && length $op; | |||
| 317 | 269 | 219 | $val //= ''; | |||
| 318 | 269 | 215 | my $lval = lc $val; # precompute once; avoids O(N) redundant lc() inside grep | |||
| 319 | return [grep { | |||||
| 320 | 269 758 | 250 745 | my $cell = $_->{$col} // ''; | |||
| 321 | 758 | 2135 | $op eq 'eq' ? lc($cell) eq $lval : | |||
| 322 | $op eq 'ne' ? lc($cell) ne $lval : | |||||
| 323 | $op eq 'contains' ? index(lc($cell), $lval) != -1 : | |||||
| 324 | $op eq 'starts' ? index(lc($cell), $lval) == 0 : | |||||
| 325 | $op eq 'lt' ? $cell < $val : | |||||
| 326 | $op eq 'le' ? $cell <= $val : | |||||
| 327 | $op eq 'gt' ? $cell > $val : | |||||
| 328 | $op eq 'ge' ? $cell >= $val : | |||||
| 329 | $op eq 'empty' ? $cell eq '' : | |||||
| 330 | $op eq 'notempty' ? $cell ne '' : | |||||
| 331 | 1 | |||||
| 332 | } @$records]; | |||||
| 333 | } | |||||
| 334 | ||||||
| 335 | # _apply_filters($self, $records) -> ($filtered_records, \@raw_specs, $json_string) | |||||
| 336 | # | |||||
| 337 | # Purpose: Read all "f=" query params, apply them in order via _apply_filter_spec, | |||||
| 338 | # and return: the filtered arrayref, the raw spec strings (for building | |||||
| 339 | # export URLs), and a script-safe JSON string for pre-populating the UI. | |||||
| 340 | # Entry: $records is an arrayref of hashrefs. | |||||
| 341 | # Exit: Three-element list; never croaks. $json_string has '</' escaped to | |||||
| 342 | # '<\/' so it is safe to embed directly in a <script> block. | |||||
| 343 | # | |||||
| 344 | # Reduction: the original code had two sequential loops over @specs -- one to | |||||
| 345 | # apply filters, one to parse them for JSON. By Modus Ponens, _apply_filter_spec | |||||
| 346 | # returns $records unchanged for a malformed spec (same guard condition), so | |||||
| 347 | # skipping the call for malformed specs is equivalent. Merging into one loop | |||||
| 348 | # eliminates a full O(n) pass and the redundant split() on every spec. | |||||
| 349 | 95 95 95 95 | 90 83 72 74 | sub _apply_filters :Protected ($self, $records) { | |||
| 350 | 95 95 | 78 237 | my @specs = @{ $self->every_param('f') // [] }; | |||
| 351 | 95 | 10793 | my @parsed; | |||
| 352 | 95 | 121 | for my $s (@specs) { | |||
| 353 | 103 | 191 | my ($col, $op, $val) = split /:/, $s, 3; | |||
| 354 | 103 | 324 | next unless length($col // '') && defined $op && length $op; | |||
| 355 | 99 | 228 | push @parsed, { col => $col, op => $op, val => $val // '' }; | |||
| 356 | 99 | 111 | $records = _apply_filter_spec($records, $s); | |||
| 357 | } | |||||
| 358 | 95 | 243 | my $json = encode_json(\@parsed); | |||
| 359 | 95 | 626 | $json =~ s{</}{<\\/}g; | |||
| 360 | 95 | 197 | return ($records, \@specs, $json); | |||
| 361 | 12 12 12 | 4540 11 99 | } | |||
| 362 | ||||||
| 363 | # _left_join($left_recs, $left_cols, $left_key, | |||||
| 364 | # $right_recs, $right_cols, $right_key, $right_label) | |||||
| 365 | # -> (\@merged_records, \@merged_columns) | |||||
| 366 | # | |||||
| 367 | # Purpose: Perform a single in-memory left join. Every left row is kept. | |||||
| 368 | # Right-table columns are appended for rows that match on the join key. | |||||
| 369 | # Unmatched rows receive undef for right-table columns. | |||||
| 370 | # If a right column name collides with a left column (other than the | |||||
| 371 | # join key itself), the right column is prefixed with "$right_label.". | |||||
| 372 | # Entry: All arrayref args non-undef; key strings non-empty; $right_label is | |||||
| 373 | # a display label used for collision-prefix (not for SQL quoting). | |||||
| 374 | # Exit: Returns (\@merged, \@column_list). | |||||
| 375 | # Side Effects: None; allocates new record hashrefs. | |||||
| 376 | sub _left_join { | |||||
| 377 | 21 | 14333 | my ($left_recs, $left_cols, $left_key, | |||
| 378 | $right_recs, $right_cols, $right_key, $right_label) = @_; | |||||
| 379 | ||||||
| 380 | # Build a lookup hash from join key to first matching right row. | |||||
| 381 | 21 | 20 | my %right_idx; | |||
| 382 | 21 | 29 | for my $row (@$right_recs) { | |||
| 383 | 55 | 65 | my $k = $row->{$right_key} // ''; | |||
| 384 | 55 | 102 | $right_idx{$k} //= $row; | |||
| 385 | } | |||||
| 386 | ||||||
| 387 | # Map right column names: drop the join key (redundant), prefix collisions. | |||||
| 388 | 21 76 | 28 75 | my %left_set = map { $_ => 1 } @$left_cols; | |||
| 389 | 21 | 24 | my (@add_cols, %col_map); | |||
| 390 | 21 59 | 27 55 | for my $col (grep { $_ ne $right_key } @$right_cols) { | |||
| 391 | 38 | 50 | my $out = $left_set{$col} ? "${right_label}.${col}" : $col; | |||
| 392 | 38 | 35 | $col_map{$col} = $out; | |||
| 393 | 38 | 39 | push @add_cols, $out; | |||
| 394 | } | |||||
| 395 | ||||||
| 396 | # Precompute [$right_col, $mapped_col] pairs once before the merge loop. | |||||
| 397 | # Without this, "grep { $_ ne $right_key } @$right_cols" would run on | |||||
| 398 | # every left row -- O(N_left * R) grep iterations for R right columns. | |||||
| 399 | # Precomputing reduces that to a single O(R) pass. | |||||
| 400 | 38 | 52 | my @rcols = map { [$_, $col_map{$_}] } | |||
| 401 | 21 59 | 24 49 | grep { $_ ne $right_key } @$right_cols; | |||
| 402 | ||||||
| 403 | 21 | 17 | my @merged; | |||
| 404 | 21 | 24 | for my $left_row (@$left_recs) { | |||
| 405 | 76 | 80 | my $k = $left_row->{$left_key} // ''; | |||
| 406 | 76 | 93 | my $right_row = $right_idx{$k} // {}; | |||
| 407 | 76 | 121 | my %row = %$left_row; | |||
| 408 | 76 | 145 | $row{ $_->[1] } = $right_row->{ $_->[0] } for @rcols; | |||
| 409 | 76 | 67 | push @merged, \%row; | |||
| 410 | } | |||||
| 411 | ||||||
| 412 | 21 | 66 | return (\@merged, [@$left_cols, @add_cols]); | |||
| 413 | } | |||||
| 414 | ||||||
| 415 | # _combine_tables(\@sources) -> (\@merged_records, \@merged_columns) | |||||
| 416 | # | |||||
| 417 | # Purpose: Perform a vertical stack (UNION ALL equivalent) of two or more | |||||
| 418 | # record sets. The merged column list is the union of all source | |||||
| 419 | # column lists: the first source's columns appear first (in their | |||||
| 420 | # original order), then each subsequent source contributes any new | |||||
| 421 | # column names it introduces, in order. Where a source does not | |||||
| 422 | # have a particular column, its rows receive an empty string for | |||||
| 423 | # that column. | |||||
| 424 | # Entry: $sources is an arrayref of [$records_aref, $columns_aref] pairs. | |||||
| 425 | # At least one element is required; a single-element call returns | |||||
| 426 | # that element's data unchanged (no allocation beyond the column copy). | |||||
| 427 | # Exit: Returns (\@merged_records, \@merged_columns). | |||||
| 428 | # Side Effects: None; allocates new record hashrefs (original rows are not mutated). | |||||
| 429 | sub _combine_tables { | |||||
| 430 | 4 | 5 | my ($sources) = @_; | |||
| 431 | ||||||
| 432 | # Build the unified column order: first source's columns first, then any | |||||
| 433 | # new columns introduced by each subsequent source in their file order. | |||||
| 434 | 4 | 5 | my (@all_cols, %seen); | |||
| 435 | 4 | 4 | for my $src (@$sources) { | |||
| 436 | 8 8 | 7 7 | for my $col (@{ $src->[1] }) { | |||
| 437 | 52 | 52 | push @all_cols, $col unless $seen{$col}++; | |||
| 438 | } | |||||
| 439 | } | |||||
| 440 | ||||||
| 441 | 4 | 3 | my @merged; | |||
| 442 | 4 | 5 | for my $src (@$sources) { | |||
| 443 | 8 | 8 | my ($recs, $cols) = @$src; | |||
| 444 | 8 52 | 8 42 | my %has_col = map { $_ => 1 } @$cols; | |||
| 445 | 8 | 10 | for my $row (@$recs) { | |||
| 446 | 96 | 49 | my %new_row; | |||
| 447 | $new_row{$_} = $has_col{$_} ? ($row->{$_} // '') : '' | |||||
| 448 | 96 | 378 | for @all_cols; | |||
| 449 | 96 | 74 | push @merged, \%new_row; | |||
| 450 | } | |||||
| 451 | } | |||||
| 452 | ||||||
| 453 | 4 | 7 | return (\@merged, \@all_cols); | |||
| 454 | } | |||||
| 455 | ||||||
| 456 | # _csv_row(@fields) -> $csv_line_with_crlf | |||||
| 457 | # | |||||
| 458 | # Purpose: Format one row as an RFC 4180 CSV line. Fields containing commas, | |||||
| 459 | # double-quotes, or newlines are enclosed in double-quotes; embedded | |||||
| 460 | # double-quotes are doubled. The line ends with CRLF. | |||||
| 461 | # Entry: @fields may contain undef (treated as empty string). | |||||
| 462 | # Exit: Returns a string ending with "\r\n". | |||||
| 463 | sub _csv_row { | |||||
| 464 | return join(',', map { | |||||
| 465 | 309 1701 | 16614 1213 | my $f = $_ // ''; | |||
| 466 | 1701 12 12 | 1787 16 24 | $f =~ /[,"\r\n]/ ? do { (my $q = $f) =~ s/"/""/g; qq{"$q"} } : $f; | |||
| 467 | } @_) . "\r\n"; | |||||
| 468 | } | |||||
| 469 | ||||||
| 470 | # _build_export_url($self, $left_spec, \@join_specs, \@filter_specs) -> $url_string | |||||
| 471 | # | |||||
| 472 | # Purpose: Assemble a /export?l=...&j=...&f=... URL string for the current | |||||
| 473 | # logical view. The caller must apply | html in TT to escape & as &. | |||||
| 474 | # Entry: All args non-undef; spec arrays may be empty. | |||||
| 475 | # Exit: Returns a relative URL string beginning with '/export'. | |||||
| 476 | 99 99 99 99 99 99 99 | 6598 118 88 74 81 91 72 | sub _build_export_url :Protected ($self, $left_spec, $join_specs, $filter_specs, $combine_specs = undef) { | |||
| 477 | 99 | 162 | my $u = '/export?l=' . url_escape($left_spec); | |||
| 478 | 99 | 1182 | $u .= '&j=' . url_escape($_) for @$join_specs; | |||
| 479 | 99 99 | 219 228 | $u .= '&c=' . url_escape($_) for @{ $combine_specs // [] }; | |||
| 480 | 99 | 162 | $u .= '&f=' . url_escape($_) for @$filter_specs; | |||
| 481 | 99 | 851 | return $u; | |||
| 482 | 12 12 12 | 6287 12 105 | } | |||
| 483 | ||||||
| 484 | # _list_dir($self, $dir, $want_files) -> { dirs => [...], files => [...] } | |||||
| 485 | # | |||||
| 486 | # Purpose: Shared directory-listing kernel used by browse (which needs files) | |||||
| 487 | # and dirs_api (which needs only subdirectories). Hidden entries | |||||
| 488 | # (names starting with ".") are excluded. Entries are sorted | |||||
| 489 | # case-insensitively. | |||||
| 490 | # Entry: $dir is a Mojo::File pointing to an existing readable directory. | |||||
| 491 | # $want_files: if true, populate 'files' with entries matching $EXT_RE. | |||||
| 492 | # Exit: Returns a hashref with 'dirs' and 'files' arrayrefs (files is always | |||||
| 493 | # present but empty when $want_files is false). | |||||
| 494 | # Side Effects: Filesystem directory read. | |||||
| 495 | 21 21 21 21 21 | 9576 21 21 20 15 | sub _list_dir :Protected ($self, $dir, $want_files) { | |||
| 496 | 21 | 23 | my (@dirs, @files); | |||
| 497 | 21 | 64 | if (opendir my $dh, $dir->to_string) { | |||
| 498 | 20 | 600 | while (my $entry = readdir $dh) { | |||
| 499 | 182 | 620 | next if $entry eq '.' || $entry eq '..' || $entry =~ /\A\./; | |||
| 500 | 83 | 81 | my $f = $dir->child($entry); | |||
| 501 | 83 | 549 | if (-d $f) { | |||
| 502 | 53 | 317 | push @dirs, { name => $entry, path => $f->to_string }; | |||
| 503 | } | |||||
| 504 | elsif ($want_files && $entry =~ $EXT_RE) { | |||||
| 505 | 15 | 162 | push @files, { name => $entry, path => $f->to_string }; | |||
| 506 | } | |||||
| 507 | } | |||||
| 508 | 20 | 131 | closedir $dh; | |||
| 509 | } | |||||
| 510 | return { | |||||
| 511 | 113 | 126 | dirs => [ sort { lc($a->{name}) cmp lc($b->{name}) } @dirs ], | |||
| 512 | 21 11 | 97 21 | files => [ sort { lc($a->{name}) cmp lc($b->{name}) } @files ], | |||
| 513 | }; | |||||
| 514 | 12 12 12 | 3047 11 89 | } | |||
| 515 | ||||||
| 516 | # _write_sqlite_db($self, $records, $columns) -> $raw_bytes | |||||
| 517 | # | |||||
| 518 | # Purpose: Write $records to an in-process SQLite database (single table named | |||||
| 519 | # "data"; every column declared TEXT) and return the raw file bytes. | |||||
| 520 | # The temporary file is created, written, read, and then unlinked in | |||||
| 521 | # one synchronous sequence. Used by both _render_sqlite (HTTP download) | |||||
| 522 | # and export_write (filesystem write) to eliminate code duplication. | |||||
| 523 | # Entry: $records is an arrayref of hashrefs; $columns is an arrayref of names. | |||||
| 524 | # Exit: Returns scalar binary string. Croaks on any DBI error. | |||||
| 525 | # Side Effects: Creates and unlinks one temporary file under system tmpdir. | |||||
| 526 | 24 24 24 24 24 | 30983 25 20 36 18 | sub _write_sqlite_db :Protected ($self, $records, $columns) { | |||
| 527 | # D~ fix: an empty column list produces "CREATE TABLE data ()" which is | |||||
| 528 | # invalid SQLite syntax. Croak before touching the filesystem. | |||||
| 529 | 24 | 67 | croak 'Cannot export SQLite: result set has no columns' | |||
| 530 | unless @$columns; | |||||
| 531 | ||||||
| 532 | 21 | 69 | require DBI; | |||
| 533 | 21 | 56 | my ($tmp_fh, $tmpfile) = tempfile(SUFFIX => '.db', UNLINK => 0); | |||
| 534 | 21 | 4128 | close $tmp_fh; | |||
| 535 | ||||||
| 536 | 21 21 | 21 107 | my $dbh = eval { DBI->connect("dbi:SQLite:dbname=$tmpfile", '', '', { | |||
| 537 | RaiseError => 1, AutoCommit => 1, | |||||
| 538 | }) }; | |||||
| 539 | # Guard both the eval-caught die ($@) and the undef-without-die edge case. | |||||
| 540 | 21 | 14560 | if ($@ || !$dbh) { | |||
| 541 | 2 | 47 | unlink $tmpfile; | |||
| 542 | 2 | 24 | croak "DBI connect failed: $@"; | |||
| 543 | } | |||||
| 544 | ||||||
| 545 | # O~ fix: wrap all DBI work in eval so any mid-flight exception still | |||||
| 546 | # triggers cleanup (disconnect + unlink) before the error propagates. | |||||
| 547 | 19 | 26 | eval { | |||
| 548 | 19 74 74 | 33 82 132 | my @quoted = map { (my $c = $_) =~ s/"/""/g; qq{"$c"} } @$columns; | |||
| 549 | 19 74 | 32 130 | $dbh->do('CREATE TABLE "data" (' . join(', ', map { "$_ TEXT" } @quoted) . ')'); | |||
| 550 | 17 | 24016 | if (@$records) { | |||
| 551 | 15 | 63 | my $ph = join(', ', ('?') x scalar @$columns); | |||
| 552 | 15 | 103 | my $sth = $dbh->prepare( | |||
| 553 | 'INSERT INTO "data" (' . join(', ', @quoted) . ") VALUES ($ph)" | |||||
| 554 | ); | |||||
| 555 | 15 | 612 | for my $row (@$records) { | |||
| 556 | 61 309 | 2878 38099 | $sth->execute(map { $row->{$_} } @$columns); | |||
| 557 | } | |||||
| 558 | } | |||||
| 559 | 17 | 1825 | $dbh->disconnect; | |||
| 560 | }; | |||||
| 561 | 19 | 53 | if (my $err = $@) { | |||
| 562 | 2 2 | 2 35 | eval { $dbh->disconnect }; # best-effort; may already be disconnected | |||
| 563 | 2 | 46 | unlink $tmpfile; | |||
| 564 | 2 | 29 | croak $err; | |||
| 565 | } | |||||
| 566 | ||||||
| 567 | 17 | 81 | my $data = Mojo::File->new($tmpfile)->slurp; | |||
| 568 | 17 | 1447 | unlink $tmpfile; | |||
| 569 | 17 | 425 | return $data; | |||
| 570 | 12 12 12 | 4225 12 93 | } | |||
| 571 | ||||||
| 572 | # _run_export_pipeline($self) -> ($records, \@columns, $left_label) or () | |||||
| 573 | # | |||||
| 574 | # Purpose: Shared join+filter pipeline executed by both export_data (GET, download) | |||||
| 575 | # and export_write (POST, filesystem write). Opens the left table, applies | |||||
| 576 | # all join steps, then applies all filter specs. | |||||
| 577 | # Entry: Reads "l=", "j=" (repeatable), and "f=" (repeatable) query/body params. | |||||
| 578 | # Exit: On success: ($filtered_arrayref, \@column_names, $left_label_string). | |||||
| 579 | # On failure: empty list (left table not found / fetch error). | |||||
| 580 | # Side Effects: Opens data files; may emit carp warnings for empty result sets. | |||||
| 581 | 62 62 62 | 60 53 49 | sub _run_export_pipeline :Protected ($self) { | |||
| 582 | 62 | 99 | my $left_spec = $self->param('l') // ''; | |||
| 583 | 62 | 6043 | my ($left_src, $left_label) = $self->_open_spec($left_spec); | |||
| 584 | 62 | 275 | return () unless $left_src; | |||
| 585 | ||||||
| 586 | 54 54 | 49 124 | my $left_recs = eval { $left_src->fetch_all }; | |||
| 587 | 54 | 87 | return () if $@; | |||
| 588 | 54 | 83 | $left_recs //= []; | |||
| 589 | ||||||
| 590 | 54 | 92 | my @columns = _get_columns($left_src, $left_recs); | |||
| 591 | 54 | 58 | my $records = $left_recs; | |||
| 592 | ||||||
| 593 | 54 54 | 47 105 | for my $jspec (@{ $self->every_param('j') }) { | |||
| 594 | 3 | 112 | my ($right_spec, $left_key, $right_key) = split /\|/, $jspec, 3; | |||
| 595 | 3 | 14 | next unless defined $right_spec && defined $left_key && defined $right_key; | |||
| 596 | 3 16 | 4 17 | my %col_set = map { $_ => 1 } @columns; | |||
| 597 | 3 | 8 | next unless $col_set{$left_key}; | |||
| 598 | 3 | 6 | my ($right_src, $right_label) = $self->_open_spec($right_spec); | |||
| 599 | 3 | 37 | next unless $right_src; | |||
| 600 | 3 3 | 4 5 | my $right_recs = eval { $right_src->fetch_all } // []; | |||
| 601 | 3 | 4 | next if $@; | |||
| 602 | 3 | 5 | my @right_cols = _get_columns($right_src, $right_recs); | |||
| 603 | 3 9 | 3 10 | my %right_set = map { $_ => 1 } @right_cols; | |||
| 604 | 3 | 9 | next unless $right_set{$right_key}; | |||
| 605 | 3 | 8 | ($records, my $new_cols) = _left_join( | |||
| 606 | $records, \@columns, $left_key, | |||||
| 607 | $right_recs, \@right_cols, $right_key, $right_label, | |||||
| 608 | ); | |||||
| 609 | 3 | 13 | @columns = @$new_cols; | |||
| 610 | } | |||||
| 611 | ||||||
| 612 | # Combine (vertical stack) with any c= sources. Runs after joins so a join | |||||
| 613 | # result can itself be stacked with another table in a single pipeline. | |||||
| 614 | 54 54 | 1944 69 | my @cspecs = @{ $self->every_param('c') // [] }; | |||
| 615 | 54 | 1287 | if (@cspecs) { | |||
| 616 | 1 | 2 | my @sources = ([$records, \@columns]); | |||
| 617 | 1 | 1 | for my $cspec (@cspecs) { | |||
| 618 | 1 | 2 | my ($csrc) = $self->_open_spec($cspec); | |||
| 619 | 1 | 3 | next unless $csrc; | |||
| 620 | 1 1 | 1 2 | my $crecs = eval { $csrc->fetch_all } // []; | |||
| 621 | 1 | 2 | next if $@; | |||
| 622 | 1 | 2 | my @ccols = _get_columns($csrc, $crecs); | |||
| 623 | 1 | 4 | push @sources, [$crecs, \@ccols]; | |||
| 624 | } | |||||
| 625 | 1 | 50 | if (@sources > 1) { | |||
| 626 | 1 | 3 | ($records, my $new_cols) = _combine_tables(\@sources); | |||
| 627 | 1 | 6 | @columns = @$new_cols; | |||
| 628 | } | |||||
| 629 | } | |||||
| 630 | ||||||
| 631 | 54 54 | 47 70 | for my $s (@{ $self->every_param('f') }) { | |||
| 632 | 10 | 178 | $records = _apply_filter_spec($records, $s); | |||
| 633 | } | |||||
| 634 | ||||||
| 635 | 54 | 1051 | return ($records, \@columns, $left_label); | |||
| 636 | 12 12 12 | 4458 10 89 | } | |||
| 637 | ||||||
| 638 | # _serialize_csv($records, \@columns) -> $utf8_string | |||||
| 639 | # | |||||
| 640 | # Purpose: Build a complete RFC 4180 CSV document (header + data rows) from | |||||
| 641 | # $records. Shared by _render_csv (HTTP download) and the csv branch | |||||
| 642 | # of export_write (filesystem write) to eliminate duplicated logic. | |||||
| 643 | # Entry: $records is an arrayref of hashrefs; $columns is an arrayref of names. | |||||
| 644 | # Exit: Returns a UTF-8 Perl string ending with CRLF. | |||||
| 645 | # | |||||
| 646 | # Optimisation: rows are collected in @lines and joined in one operation. | |||||
| 647 | # The previous pattern ($out .= _csv_row(...) per row) triggers O(N) string | |||||
| 648 | # reallocations; joining a pre-built array is a single O(total_bytes) allocation. | |||||
| 649 | # For exports of thousands of rows the difference is measurable. | |||||
| 650 | # | |||||
| 651 | # XS note: for very large exports, replacing _csv_row with Text::CSV_XS | |||||
| 652 | # (available from CPAN) would further reduce serialisation time -- its C-level | |||||
| 653 | # implementation is roughly 5-10x faster than the pure-Perl version here. | |||||
| 654 | sub _serialize_csv { | |||||
| 655 | 50 | 9123 | my ($records, $columns) = @_; | |||
| 656 | 50 | 95 | my @lines = _csv_row(@$columns); | |||
| 657 | 50 | 72 | for my $row (@$records) { | |||
| 658 | 239 1399 | 188 1087 | push @lines, _csv_row(map { $row->{$_} } @$columns); | |||
| 659 | } | |||||
| 660 | 50 | 143 | return encode('UTF-8', join('', @lines)); | |||
| 661 | } | |||||
| 662 | ||||||
| 663 | # _render_csv($self, $records, \@columns, $name) -> void (renders response) | |||||
| 664 | # | |||||
| 665 | # Purpose: Serialise $records to RFC 4180 CSV and emit as a UTF-8 download. | |||||
| 666 | # Entry: $name is a safe filename stem (alphanumeric/underscore). | |||||
| 667 | # Exit: Renders the Mojolicious response; does not return a meaningful value. | |||||
| 668 | # Side Effects: Writes HTTP response headers and body. | |||||
| 669 | 27 27 27 27 27 27 | 28 27 28 21 25 21 | sub _render_csv :Protected ($self, $records, $columns, $name) { | |||
| 670 | 27 | 45 | $self->res->headers->content_type('text/csv; charset=UTF-8'); | |||
| 671 | 27 | 316 | $self->res->headers->content_disposition(qq{attachment; filename="${name}.csv"}); | |||
| 672 | 27 | 273 | $self->render(data => _serialize_csv($records, $columns)); | |||
| 673 | 12 12 12 | 2375 7 86 | } | |||
| 674 | ||||||
| 675 | # _render_sqlite($self, $records, \@columns, $name) -> void (renders response) | |||||
| 676 | # | |||||
| 677 | # Purpose: Write $records to a SQLite database and emit it as a binary download. | |||||
| 678 | # Entry: $name is a safe filename stem. | |||||
| 679 | # Exit: Renders the Mojolicious response. | |||||
| 680 | # Side Effects: Creates and unlinks a temporary file; writes HTTP response. | |||||
| 681 | 6 6 6 6 6 6 | 7 7 8 5 6 6 | sub _render_sqlite :Protected ($self, $records, $columns, $name) { | |||
| 682 | 6 | 19 | my $data = $self->_write_sqlite_db($records, $columns); | |||
| 683 | 6 | 21 | $self->res->headers->content_type('application/vnd.sqlite3'); | |||
| 684 | 6 | 120 | $self->res->headers->content_disposition(qq{attachment; filename="${name}.db"}); | |||
| 685 | 6 | 71 | $self->render(data => $data); | |||
| 686 | 12 12 12 | 1607 43 95 | } | |||
| 687 | ||||||
| 688 | # --------------------------------------------------------------------------- | |||||
| 689 | # Public actions | |||||
| 690 | # --------------------------------------------------------------------------- | |||||
| 691 | ||||||
| 692 - 727 | =head1 ACTIONS
=head2 index
C<GET /> -- Scan C<data_dir> and present a card grid of available tables.
=head3 API SPECIFICATION
=head4 INPUT
None (reads C<data_dir> from application config).
=head4 OUTPUT
Renders C<home.html.tt> with:
tables ARRAYREF of { name => $stem, file => $basename }
title 'Choose a Database'
=head3 MESSAGES
None produced by this action; any file-scan errors are silently ignored
(empty directory yields an empty card grid).
=head3 FORMAL SPECIFICATION
index == lambda self .
let dir = resolve self.app.config.data_dir in
let tbls = { b | b in dir /\ basename(b) =~ EXT_RE } in
post render(home, tables: map(stem, tbls))
=head3 EXAMPLE
GET / -> 200 text/html containing a card for each file in data/
=cut | |||||
| 728 | ||||||
| 729 | 16 16 16 | 118736 19 11 | sub index ($self) { | |||
| 730 | 16 | 45 | my ($platform, $language) = $self->_resolve_template; | |||
| 731 | ||||||
| 732 | 16 | 44 | $self->render( | |||
| 733 | template => "$platform/$language/home", | |||||
| 734 | handler => 'tt', | |||||
| 735 | format => 'html', | |||||
| 736 | tables => $self->_scan_data_dir, | |||||
| 737 | title => 'Choose a Database', | |||||
| 738 | ); | |||||
| 739 | } | |||||
| 740 | ||||||
| 741 - 804 | =head2 view
C<GET /view/:table> -- Open and display the chosen table from C<data_dir>.
=head3 API SPECIFICATION
=head4 INPUT
:table string Table name (alphanumeric + underscore). Returns 404 on
other characters.
?f= string (repeatable) Filter spec: "col:op:val".
=head4 DOMAIN CONSTRAINTS: :table
C<:table> is validated against C<\A[A-Za-z_][A-Za-z0-9_]*\z> before C<open_table>
is ever called. The first character must be a letter (A-Z, a-z) or an
underscore; subsequent characters may also include digits (0-9).
=over 4
=item Valid partition
C<sales> (letters only), C<_temp> (underscore-start), C<report_2024>
(mixed). A name that is valid but has no backing file returns 200 with
an error message -- NOT 404.
=item Invalid partition
C<1sales> (digit-start, 404), C<my.data> (dot, 404), C<my-data>
(hyphen, 404), non-ASCII characters (404).
=item Boundary values
C<a> (single letter, valid), C<_> (single underscore, valid), C<1>
(single digit, 404), C<a1> (letter then digit, valid), C<1a> (digit
then letter, 404).
=back
=head4 OUTPUT
On success renders C<dashboard.html.tt> with table data.
On error re-renders C<home.html.tt> with an C<error> stash variable.
=head3 MESSAGES
error_table_open -- DataSource initialisation or fetch_all threw.
=head3 FORMAL SPECIFICATION
view == lambda self .
pre self.stash('table') =~ TABLE_NAME_RE
let src = open_table(table) in
let records = src.fetch_all() in
let cols = get_columns(src, records) in
post render(dashboard, records: filter(records, f_params))
=head3 EXAMPLE
GET /view/sales -> 200 HTML table of all sales rows
GET /view/sales?f=region:eq:North -> shows only North region rows
GET /view/../etc -> 404
=cut | |||||
| 805 | ||||||
| 806 | 75 75 75 | 897418 82 63 | sub view ($self) { | |||
| 807 | 75 | 146 | my ($platform, $language) = $self->_resolve_template; | |||
| 808 | 75 | 106 | my $table = $self->stash('table'); | |||
| 809 | ||||||
| 810 | 75 | 580 | unless (defined $table && $table =~ $TABLE_NAME_RE) { | |||
| 811 | 14 | 102 | return $self->reply->not_found; | |||
| 812 | } | |||||
| 813 | ||||||
| 814 | 61 | 353 | my ($source, $records); | |||
| 815 | 61 61 61 | 56 145 141 | eval { $source = $self->open_table($table); $records = $source->fetch_all }; | |||
| 816 | 61 | 3543 | if ($@) { | |||
| 817 | 9 | 52 | return $self->render( | |||
| 818 | template => "$platform/$language/home", | |||||
| 819 | handler => 'tt', | |||||
| 820 | format => 'html', | |||||
| 821 | tables => [], | |||||
| 822 | title => 'Choose a Database', | |||||
| 823 | error => $self->_i18n('error_table_open', $table, $@), | |||||
| 824 | ); | |||||
| 825 | } | |||||
| 826 | ||||||
| 827 | 52 | 124 | my @columns = _get_columns($source, $records); | |||
| 828 | 52 | 128 | my ($filtered, $filter_specs, $filters_json) = $self->_apply_filters($records); | |||
| 829 | ||||||
| 830 | 52 | 229 | $self->render( | |||
| 831 | template => "$platform/$language/dashboard", | |||||
| 832 | handler => 'tt', | |||||
| 833 | format => 'html', | |||||
| 834 | records => $filtered, | |||||
| 835 | columns => \@columns, | |||||
| 836 | table => $table, | |||||
| 837 | title => ucfirst($table), | |||||
| 838 | left_spec => "table:$table", | |||||
| 839 | combine_specs => [], | |||||
| 840 | current_joins => [], | |||||
| 841 | available_tables => $self->_scan_data_dir, | |||||
| 842 | join_summaries => [], | |||||
| 843 | filter_specs => $filter_specs, | |||||
| 844 | filters_json => $filters_json, | |||||
| 845 | export_url => $self->_build_export_url("table:$table", [], $filter_specs), | |||||
| 846 | ); | |||||
| 847 | } | |||||
| 848 | ||||||
| 849 - 888 | =head2 browse
C<GET /browse> -- Navigate the filesystem and pick a data file.
=head3 API SPECIFICATION
=head4 INPUT
?path= string Absolute filesystem path to browse (default: $HOME).
Returns 404 when the path does not exist or is not a
directory.
=head4 OUTPUT
Renders C<browse.html.tt> with:
current_path string
parent_href string or undef (undef at filesystem root)
crumbs ARRAYREF of { name, href }
dirs ARRAYREF of { name, href } -- subdirectories, sorted
files ARRAYREF of { name, href } -- supported data files, sorted
=head3 MESSAGES
None; errors render as 404.
=head3 FORMAL SPECIFICATION
browse == lambda self .
let dir = realpath(param('path') // HOME) in
pre is_dir(dir)
post render(browse, dirs: subdirs(dir), files: supported_files(dir))
=head3 EXAMPLE
GET /browse -> 200, lists $HOME
GET /browse?path=/tmp -> 200, lists /tmp
GET /browse?path=/nonexistent/xyz -> 404
=cut | |||||
| 889 | ||||||
| 890 | 16 16 16 | 138067 16 14 | sub browse ($self) { | |||
| 891 | 16 | 32 | my ($platform, $language) = $self->_resolve_template; | |||
| 892 | ||||||
| 893 | 16 | 40 | my $raw_path = $self->param('path') // $ENV{HOME} // '/'; | |||
| 894 | 16 16 | 1894 25 | my $dir = eval { Mojo::File->new($raw_path)->realpath }; | |||
| 895 | 16 | 1553 | return $self->reply->not_found unless defined $dir && -d $dir; | |||
| 896 | ||||||
| 897 | 9 | 92 | my $listing = $self->_list_dir($dir, 1); | |||
| 898 | ||||||
| 899 | # Add browse hrefs to file entries (dirs already have path; browse needs href). | |||||
| 900 | my @dirs = map { | |||||
| 901 | 13 | 92 | +{ name => $_->{name}, href => '/browse?path=' . url_escape($_->{path}) } | |||
| 902 | 9 9 | 12 14 | } @{ $listing->{dirs} }; | |||
| 903 | ||||||
| 904 | my @files = map { | |||||
| 905 | 12 | 100 | +{ name => $_->{name}, href => '/open?path=' . url_escape($_->{path}) } | |||
| 906 | 9 9 | 39 12 | } @{ $listing->{files} }; | |||
| 907 | ||||||
| 908 | # Build breadcrumb trail from filesystem root down to $dir. | |||||
| 909 | 9 | 67 | my @crumbs; | |||
| 910 | { | |||||
| 911 | 9 9 | 8 9 | my $f = $dir; | |||
| 912 | 9 | 8 | while (1) { | |||
| 913 | 35 | 43 | my $name = $f->basename; | |||
| 914 | 35 | 438 | $name = '/' unless length $name; | |||
| 915 | 35 | 44 | unshift @crumbs, { name => $name, href => '/browse?path=' . url_escape($f->to_string) }; | |||
| 916 | 35 | 304 | my $parent = $f->dirname; | |||
| 917 | 35 | 487 | last if $parent->to_string eq $f->to_string; | |||
| 918 | 26 | 95 | $f = $parent; | |||
| 919 | } | |||||
| 920 | } | |||||
| 921 | ||||||
| 922 | 9 | 56 | my $parent = $dir->dirname; | |||
| 923 | 9 | 98 | my $parent_href = $parent->to_string ne $dir->to_string | |||
| 924 | ? '/browse?path=' . url_escape($parent->to_string) | |||||
| 925 | : undef; | |||||
| 926 | ||||||
| 927 | 9 | 108 | $self->render( | |||
| 928 | template => "$platform/$language/browse", | |||||
| 929 | handler => 'tt', | |||||
| 930 | format => 'html', | |||||
| 931 | title => 'Browse Files', | |||||
| 932 | current_path => $dir->to_string, | |||||
| 933 | parent_href => $parent_href, | |||||
| 934 | crumbs => \@crumbs, | |||||
| 935 | dirs => \@dirs, | |||||
| 936 | files => \@files, | |||||
| 937 | ); | |||||
| 938 | } | |||||
| 939 | ||||||
| 940 - 1010 | =head2 open_file
C<GET /open> -- Open a supported data file from any absolute filesystem path.
=head3 API SPECIFICATION
=head4 INPUT
?path= string Absolute path to the data file. Returns 404 when missing,
not a regular file, or the extension is not in SUPPORTED_EXT.
?f= string (repeatable) Filter spec: "col:op:val".
=head4 DOMAIN CONSTRAINTS: ?path=
=over 4
=item Extension filter (C<EXT_RE>)
The basename must match C<\.(?:csv|db|sql|xml|psv)\z> (case-insensitive).
The C<\z> anchor (absolute end-of-string) means a URL-encoded trailing
newline (e.g. C<file.csv%0A> decoded to C<file.csv\n>) does NOT pass --
the C<\n> falls after the C<\z> boundary and the extension check fails.
=item Valid partition
C</data/sales.csv> (lowercase extension), C</tmp/REPORT.CSV> (uppercase
extension, /i matches), any C<.db>, C<.sql>, C<.xml>, C<.psv> regular
file.
=item Invalid partition
Absent C<?path=> (404), non-existent file (404), directory instead of
file (404), unsupported extension such as C<.txt> (404), empty string
(404), path containing a C<%0A> (newline) suffix (404).
=item Double-extension filenames
A file like C<file.php.csv> passes C<EXT_RE> (last segment is C<.csv>)
but produces table stem C<file.php> which fails C<DataSource>'s
C<TABLE_NAME_RE>. The C<open_table> call is inside C<eval>, so the
croak is caught and the action returns 200 with a friendly error, not
a 500.
=back
=head4 OUTPUT
On success renders C<dashboard.html.tt> with C<back_url> pointing to the
directory's C</browse> page, and C<file_path> set to the resolved absolute
path (used by the template to record the file in C<localStorage>).
On error re-renders C<home.html.tt> with C<error>, C<back_url>, C<back_label>.
=head3 MESSAGES
error_file_open -- DataSource threw during initialisation or fetch_all.
=head3 FORMAL SPECIFICATION
open_file == lambda self .
let file = realpath(param('path')) in
pre is_file(file) /\ basename(file) =~ EXT_RE
let src = open_table(stem(file), dir=dirname(file)) in
post render(dashboard, file_path: file.to_string)
=head3 EXAMPLE
GET /open?path=/data/archive/sales.csv -> 200, table view
GET /open -> 404
GET /open?path=/etc/passwd -> 404
=cut | |||||
| 1011 | ||||||
| 1012 | 53 53 53 | 485184 58 49 | sub open_file ($self) { | |||
| 1013 | 53 | 144 | my ($platform, $language) = $self->_resolve_template; | |||
| 1014 | 53 | 105 | my $file_path = $self->param('path'); | |||
| 1015 | ||||||
| 1016 | 53 | 6574 | return $self->reply->not_found unless defined $file_path; | |||
| 1017 | ||||||
| 1018 | 48 48 | 42 81 | my $file = eval { Mojo::File->new($file_path)->realpath }; | |||
| 1019 | 48 | 2657 | return $self->reply->not_found | |||
| 1020 | unless defined $file && -f $file && $file->basename =~ $EXT_RE; | |||||
| 1021 | ||||||
| 1022 | 31 | 1228 | my $dir = $file->dirname; | |||
| 1023 | 31 | 639 | (my $table = $file->basename) =~ s/\.[^.]+\z//; | |||
| 1024 | 31 | 405 | $table = lc $table; | |||
| 1025 | 31 | 47 | my $back = '/browse?path=' . url_escape($dir->to_string); | |||
| 1026 | 31 | 449 | my $filename = $file->basename; | |||
| 1027 | 31 | 317 | my $lspec = 'path:' . $file->to_string; | |||
| 1028 | ||||||
| 1029 | 31 | 80 | my ($source, $records); | |||
| 1030 | 31 31 26 | 26 48 60 | eval { $source = $self->open_table($table, directory => $dir->to_string); $records = $source->fetch_all }; | |||
| 1031 | 31 | 4150 | if ($@) { | |||
| 1032 | 9 | 54 | return $self->render( | |||
| 1033 | template => "$platform/$language/home", | |||||
| 1034 | handler => 'tt', | |||||
| 1035 | format => 'html', | |||||
| 1036 | tables => [], | |||||
| 1037 | title => 'Error', | |||||
| 1038 | error => $self->_i18n('error_file_open', $filename, $@), | |||||
| 1039 | back_url => $back, | |||||
| 1040 | back_label => 'Back to browser', | |||||
| 1041 | ); | |||||
| 1042 | } | |||||
| 1043 | ||||||
| 1044 | 22 | 48 | my @columns = _get_columns($source, $records); | |||
| 1045 | 22 | 63 | my ($filtered, $filter_specs, $filters_json) = $self->_apply_filters($records); | |||
| 1046 | ||||||
| 1047 | 22 | 88 | $self->render( | |||
| 1048 | template => "$platform/$language/dashboard", | |||||
| 1049 | handler => 'tt', | |||||
| 1050 | format => 'html', | |||||
| 1051 | records => $filtered, | |||||
| 1052 | columns => \@columns, | |||||
| 1053 | table => $table, | |||||
| 1054 | title => $filename, | |||||
| 1055 | back_url => $back, | |||||
| 1056 | back_label => 'Back to browser', | |||||
| 1057 | file_path => $file->to_string, | |||||
| 1058 | left_spec => $lspec, | |||||
| 1059 | combine_specs => [], | |||||
| 1060 | current_joins => [], | |||||
| 1061 | available_tables => $self->_scan_data_dir, | |||||
| 1062 | join_summaries => [], | |||||
| 1063 | filter_specs => $filter_specs, | |||||
| 1064 | filters_json => $filters_json, | |||||
| 1065 | export_url => $self->_build_export_url($lspec, [], $filter_specs), | |||||
| 1066 | ); | |||||
| 1067 | } | |||||
| 1068 | ||||||
| 1069 - 1104 | =head2 import_url C<GET /import> -- Fetch a remote HTML page, extract the first (or selected) table, and render it as a sortable data grid. =head3 API SPECIFICATION =head4 INPUT ?url= string Required. Full http:// or https:// URL of the page that contains the HTML table. Returns home with an error message if the scheme is wrong, the fetch fails, or no table is found at the specified index. ?table_index= integer Zero-based index of the C<< <table> >> to use when the page contains more than one table (default: 0). ?f= string (repeatable) Filter spec: "col:op:val". =head4 OUTPUT On success renders C<dashboard.html.tt> with the same stash shape as C<open_file>, plus C<source_url> (the original URL) for C<localStorage> tracking. On error re-renders C<home.html.tt> with an C<error> stash variable. =head3 MESSAGES error_url_required -- empty or missing C<url> param error_url_invalid -- URL does not begin with http:// or https:// error_url_fetch -- LWP fetch failure or no table found at the index =head3 EXAMPLE GET /import?url=https://matrix.perl-magpie.org/dist/Database-Abstraction/ GET /import?url=https://example.com/data.html&table_index=2 =cut | |||||
| 1105 | ||||||
| 1106 | 28 28 28 | 276250 34 20 | sub import_url ($self) { | |||
| 1107 | 28 | 54 | my ($platform, $language) = $self->_resolve_template; | |||
| 1108 | ||||||
| 1109 | 28 | 52 | my $url = $self->param('url') // ''; | |||
| 1110 | 28 | 3434 | my $idx = $self->param('table_index') // 0; | |||
| 1111 | 28 | 663 | $idx = 0 unless $idx =~ /\A[0-9]+\z/; | |||
| 1112 | ||||||
| 1113 | 25 25 25 25 | 30 23 25 22 | my $err_home = sub ($key, @args) { | |||
| 1114 | 25 | 55 | $self->render( | |||
| 1115 | template => "$platform/$language/home", | |||||
| 1116 | handler => 'tt', | |||||
| 1117 | format => 'html', | |||||
| 1118 | tables => $self->_scan_data_dir, | |||||
| 1119 | title => 'Choose a Database', | |||||
| 1120 | error => $self->_i18n($key, @args), | |||||
| 1121 | ); | |||||
| 1122 | 28 | 60 | }; | |||
| 1123 | ||||||
| 1124 | 28 | 49 | return $err_home->('error_url_required') unless length $url; | |||
| 1125 | 23 | 113 | return $err_home->('error_url_invalid', $url) | |||
| 1126 | unless $url =~ m{\Ahttps?://}i; | |||||
| 1127 | 17 | 36 | return $err_home->('error_url_ssrf', $url) | |||
| 1128 | unless _is_safe_url($url); | |||||
| 1129 | ||||||
| 1130 | 6 6 | 8 14 | my $source = eval { $self->open_table('', url => $url, html_table_index => $idx) }; | |||
| 1131 | 6 | 401 | return $err_home->('error_url_fetch', $url, $@ // 'unknown error') if $@ || !$source; | |||
| 1132 | ||||||
| 1133 | 5 5 | 5 10 | my $records = eval { $source->fetch_all }; | |||
| 1134 | 5 | 432 | return $err_home->('error_url_fetch', $url, $@ // 'empty result') if $@; | |||
| 1135 | ||||||
| 1136 | 3 | 8 | my $label = $source->table_name; | |||
| 1137 | 3 | 4 | my $lspec = "url:$url"; | |||
| 1138 | 3 | 8 | my @columns = _get_columns($source, $records); | |||
| 1139 | 3 | 7 | my ($filtered, $filter_specs, $filters_json) = $self->_apply_filters($records); | |||
| 1140 | ||||||
| 1141 | 3 | 10 | $self->render( | |||
| 1142 | template => "$platform/$language/dashboard", | |||||
| 1143 | handler => 'tt', | |||||
| 1144 | format => 'html', | |||||
| 1145 | records => $filtered, | |||||
| 1146 | columns => \@columns, | |||||
| 1147 | table => $label, | |||||
| 1148 | title => $label, | |||||
| 1149 | source_url => $url, | |||||
| 1150 | back_url => '/', | |||||
| 1151 | back_label => 'Choose another database', | |||||
| 1152 | left_spec => $lspec, | |||||
| 1153 | combine_specs => [], | |||||
| 1154 | current_joins => [], | |||||
| 1155 | available_tables => $self->_scan_data_dir, | |||||
| 1156 | join_summaries => [], | |||||
| 1157 | filter_specs => $filter_specs, | |||||
| 1158 | filters_json => $filters_json, | |||||
| 1159 | export_url => $self->_build_export_url($lspec, [], $filter_specs), | |||||
| 1160 | ); | |||||
| 1161 | } | |||||
| 1162 | ||||||
| 1163 - 1200 | =head2 columns_api
C<GET /api/columns> -- Return column names for a table or file as JSON.
Used by the join UI to populate the right-key dropdown without a page reload.
=head3 API SPECIFICATION
=head4 INPUT
?table= string Table name from C<data_dir>.
?path= string Absolute path to a data file.
One of C<table> or C<path> must be present; if both are, C<table> takes
precedence.
=head4 OUTPUT
200 application/json { "columns": ["col1", "col2", ...] }
404 application/json { "error": "not found" }
=head3 MESSAGES
None produced by this action directly; internal errors yield a 404.
=head3 FORMAL SPECIFICATION
columns_api == lambda self .
let src = open_spec(table_param or path_param) in
pre src /= undef
post render_json({ columns: get_columns(src) })
=head3 EXAMPLE
GET /api/columns?table=sales -> {"columns":["product","region","amount"]}
GET /api/columns?table=noexist -> 404
=cut | |||||
| 1201 | ||||||
| 1202 | 14 14 14 | 99509 12 14 | sub columns_api ($self) { | |||
| 1203 | 14 | 33 | my $table_name = $self->param('table'); | |||
| 1204 | 14 | 1680 | my $path = $self->param('path'); | |||
| 1205 | ||||||
| 1206 | 14 | 305 | my $source; | |||
| 1207 | 14 | 87 | if (defined $table_name && $table_name =~ $TABLE_NAME_RE) { | |||
| 1208 | 9 | 68 | my $data_dir = $self->app->home->child($self->app->config->{data_dir} // 'data'); | |||
| 1209 | return $self->render(json => { error => 'not found' }, status => 404) | |||||
| 1210 | 9 45 | 189 577 | unless grep { -f $data_dir->child(lc($table_name) . ".$_") } @SUPPORTED_EXT; | |||
| 1211 | 7 7 | 89 21 | $source = eval { $self->open_table(lc $table_name, directory => $data_dir->to_string) }; | |||
| 1212 | } | |||||
| 1213 | elsif (defined $path) { | |||||
| 1214 | 3 3 | 5 6 | my $file = eval { Mojo::File->new($path)->realpath }; | |||
| 1215 | 3 | 90 | if (defined $file && -f $file && $file->basename =~ $EXT_RE) { | |||
| 1216 | 2 | 88 | my $dir = $file->dirname->to_string; | |||
| 1217 | 2 | 53 | (my $tbl = $file->basename) =~ s/\.[^.]+\z//; | |||
| 1218 | 2 2 | 26 6 | $source = eval { $self->open_table(lc $tbl, directory => $dir) }; | |||
| 1219 | } | |||||
| 1220 | } | |||||
| 1221 | ||||||
| 1222 | 12 | 70 | return $self->render(json => { error => 'not found' }, status => 404) unless $source; | |||
| 1223 | ||||||
| 1224 | # Use _get_columns to ensure consistent ordering with the view action. | |||||
| 1225 | 9 1 | 20 1 | my $recs = ($source->columns ? [] : eval { $source->fetch_all } // []); | |||
| 1226 | 9 | 21 | my @cols = _get_columns($source, $recs); | |||
| 1227 | ||||||
| 1228 | 9 | 29 | $self->render(json => { columns => \@cols }); | |||
| 1229 | } | |||||
| 1230 | ||||||
| 1231 - 1280 | =head2 join_tables
C<GET /join> -- Perform one or more left joins and render the merged table.
=head3 API SPECIFICATION
=head4 INPUT
?l= string Required. Left table spec: "table:name" or "path:/abs/path".
Returns 404 if unresolvable.
?j= string (repeatable) Join step: "<right-spec>|<left-key>|<right-key>".
Invalid or non-existent steps are silently skipped.
?f= string (repeatable) Result filter: "col:op:val".
=head4 OUTPUT
Renders C<dashboard.html.tt> with the merged/filtered result set.
=head3 MESSAGES
error_table_open -- Left table fetch threw (re-renders home with error).
=head3 FORMAL SPECIFICATION
join_tables == lambda self .
let left = open_spec(param('l')) in
pre left /= undef
let joined = fold(_left_join, left, param_list('j')) in
let filtered = fold(_apply_filter_spec, joined, param_list('f')) in
post render(dashboard, records: filtered)
=head3 PSEUDOCODE
1. Parse and open left table spec; 404 on failure.
2. For each j= param (in order):
a. Parse right-spec, left-key, right-key from "|"-split.
b. Verify left-key exists in current column list.
c. Open right table; skip step on any error.
d. Verify right-key exists in right column list.
e. Call _left_join; update records and columns.
3. Apply all f= filters via _apply_filter_spec.
4. Build stable table key for localStorage ("join:left:right1:...").
5. Render dashboard.
=head3 EXAMPLE
GET /join?l=table:sales&j=table:products|product|name
-> merged left join of sales and products on product/name columns
=cut | |||||
| 1281 | ||||||
| 1282 | 22 22 22 | 187095 23 21 | sub join_tables ($self) { | |||
| 1283 | 22 | 51 | my ($platform, $language) = $self->_resolve_template; | |||
| 1284 | ||||||
| 1285 | 22 | 49 | my $left_spec = $self->param('l') // ''; | |||
| 1286 | 22 | 3102 | my ($left_src, $left_label) = $self->_open_spec($left_spec); | |||
| 1287 | 22 | 125 | return $self->reply->not_found unless $left_src; | |||
| 1288 | ||||||
| 1289 | 15 15 | 14 34 | my $left_recs = eval { $left_src->fetch_all }; | |||
| 1290 | 15 | 29 | if ($@) { | |||
| 1291 | 0 | 0 | return $self->render( | |||
| 1292 | template => "$platform/$language/home", | |||||
| 1293 | handler => 'tt', | |||||
| 1294 | format => 'html', | |||||
| 1295 | tables => [], | |||||
| 1296 | title => 'Error', | |||||
| 1297 | error => $self->_i18n('error_table_open', $left_label, $@), | |||||
| 1298 | ); | |||||
| 1299 | } | |||||
| 1300 | 15 | 34 | $left_recs //= []; | |||
| 1301 | ||||||
| 1302 | 15 | 34 | my @left_cols = _get_columns($left_src, $left_recs); | |||
| 1303 | 15 15 | 21 30 | my @join_specs = @{ $self->every_param('j') }; | |||
| 1304 | ||||||
| 1305 | 15 | 527 | my ($records, @columns, @summaries) = ($left_recs, @left_cols); | |||
| 1306 | ||||||
| 1307 | 15 | 26 | for my $jspec (@join_specs) { | |||
| 1308 | 10 | 35 | my ($right_spec, $left_key, $right_key) = split /\|/, $jspec, 3; | |||
| 1309 | 10 | 45 | next unless defined $right_spec && defined $left_key && defined $right_key; | |||
| 1310 | ||||||
| 1311 | # O(1) hash-set probe instead of O(C) grep for each join step. | |||||
| 1312 | 10 53 | 13 54 | my %col_set = map { $_ => 1 } @columns; | |||
| 1313 | 10 | 22 | next unless $col_set{$left_key}; | |||
| 1314 | ||||||
| 1315 | 10 | 21 | my ($right_src, $right_label) = $self->_open_spec($right_spec); | |||
| 1316 | 10 | 86 | next unless $right_src; | |||
| 1317 | ||||||
| 1318 | 8 8 | 7 15 | my $right_recs = eval { $right_src->fetch_all } // []; | |||
| 1319 | 8 | 17 | next if $@; | |||
| 1320 | 8 | 12 | my @right_cols = _get_columns($right_src, $right_recs); | |||
| 1321 | 8 29 | 12 29 | my %right_set = map { $_ => 1 } @right_cols; | |||
| 1322 | 8 | 22 | next unless $right_set{$right_key}; | |||
| 1323 | ||||||
| 1324 | 8 | 24 | ($records, my $new_cols) = _left_join( | |||
| 1325 | $records, \@columns, $left_key, | |||||
| 1326 | $right_recs, \@right_cols, $right_key, $right_label, | |||||
| 1327 | ); | |||||
| 1328 | 8 | 16 | @columns = @$new_cols; | |||
| 1329 | 8 | 45 | push @summaries, { label => $right_label, left_key => $left_key, right_key => $right_key }; | |||
| 1330 | } | |||||
| 1331 | ||||||
| 1332 | 15 | 473 | my ($filtered, $filter_specs, $filters_json) = $self->_apply_filters($records); | |||
| 1333 | ||||||
| 1334 | 15 | 21 | my $title = $left_label; | |||
| 1335 | 15 8 | 28 17 | $title .= ' + ' . join(' + ', map { $_->{label} } @summaries) if @summaries; | |||
| 1336 | 15 | 26 | my $table_key = 'join:' . lc($left_label); | |||
| 1337 | 15 | 33 | $table_key .= ':' . lc($_->{label}) for @summaries; | |||
| 1338 | ||||||
| 1339 | 15 | 52 | $self->render( | |||
| 1340 | template => "$platform/$language/dashboard", | |||||
| 1341 | handler => 'tt', | |||||
| 1342 | format => 'html', | |||||
| 1343 | records => $filtered, | |||||
| 1344 | columns => \@columns, | |||||
| 1345 | table => $table_key, | |||||
| 1346 | title => $title, | |||||
| 1347 | back_url => $self->_spec_to_url($left_spec), | |||||
| 1348 | back_label => "Back to $left_label", | |||||
| 1349 | left_spec => $left_spec, | |||||
| 1350 | combine_specs => [], | |||||
| 1351 | current_joins => \@join_specs, | |||||
| 1352 | available_tables => $self->_scan_data_dir, | |||||
| 1353 | join_summaries => \@summaries, | |||||
| 1354 | filter_specs => $filter_specs, | |||||
| 1355 | filters_json => $filters_json, | |||||
| 1356 | export_url => $self->_build_export_url($left_spec, \@join_specs, $filter_specs), | |||||
| 1357 | ); | |||||
| 1358 | } | |||||
| 1359 | ||||||
| 1360 - 1409 | =head2 combine_tables
C<GET /combine> -- Stack rows from two or more tables vertically into a single
unified view.
Unlike C</join> (which appends columns from a matching row in a second table),
C</combine> appends the I<rows> from a second table beneath the rows of the
first. All columns from all sources appear as headers; where a row has no
value for a particular column (because that column did not exist in its source
file) the cell is left blank.
This is equivalent to a SQL C<UNION ALL> across heterogeneous schemas.
=head3 API SPECIFICATION
=head4 INPUT
?l= string (required) Left table spec: "table:name" or "path:/abs/path".
Returns 404 if unresolvable.
?c= string (repeatable) Additional table spec to stack beneath the
current result. Invalid or non-existent specs are silently
skipped.
?f= string (repeatable) Result filter: "col:op:val".
=head4 OUTPUT
Renders C<dashboard.html.tt> with the combined result set.
=head3 MESSAGES
error_table_open -- Left table fetch threw (re-renders home with error).
=head3 FORMAL SPECIFICATION
combine_tables == lambda self .
let left = open_spec(param('l')) in
pre left /= undef
let sources = [left] ++ map(open_spec, param_list('c')) in
let combined = _combine_tables(sources) in
let filtered = fold(_apply_filter_spec, combined, param_list('f')) in
post render(dashboard, records: filtered)
=head3 EXAMPLE
GET /combine?l=table:cats&c=table:dogs
-> unified view: all cat and dog rows; Species/Name/Color/Breed/Eye Color
appear for both; Environment (cats-only) and Sex/Fixed (dogs-only)
are blank where the source file lacks that column.
=cut | |||||
| 1410 | ||||||
| 1411 | 3 3 3 | 19217 3 2 | sub combine_tables ($self) { | |||
| 1412 | 3 | 9 | my ($platform, $language) = $self->_resolve_template; | |||
| 1413 | ||||||
| 1414 | 3 | 6 | my $left_spec = $self->param('l') // ''; | |||
| 1415 | 3 | 447 | my ($left_src, $left_label) = $self->_open_spec($left_spec); | |||
| 1416 | 3 | 5 | return $self->reply->not_found unless $left_src; | |||
| 1417 | ||||||
| 1418 | 3 3 | 2 6 | my $left_recs = eval { $left_src->fetch_all }; | |||
| 1419 | 3 | 6 | if ($@) { | |||
| 1420 | 0 | 0 | return $self->render( | |||
| 1421 | template => "$platform/$language/home", | |||||
| 1422 | handler => 'tt', | |||||
| 1423 | format => 'html', | |||||
| 1424 | tables => [], | |||||
| 1425 | title => 'Error', | |||||
| 1426 | error => $self->_i18n('error_table_open', $left_label, $@), | |||||
| 1427 | ); | |||||
| 1428 | } | |||||
| 1429 | 3 | 5 | $left_recs //= []; | |||
| 1430 | ||||||
| 1431 | 3 | 5 | my @left_cols = _get_columns($left_src, $left_recs); | |||
| 1432 | 3 3 | 3 5 | my @combine_specs = @{ $self->every_param('c') }; | |||
| 1433 | ||||||
| 1434 | 3 | 120 | my @sources = ([$left_recs, \@left_cols]); | |||
| 1435 | 3 | 3 | my @summaries; | |||
| 1436 | ||||||
| 1437 | 3 | 57 | for my $cspec (@combine_specs) { | |||
| 1438 | 3 | 5 | my ($csrc, $clabel) = $self->_open_spec($cspec); | |||
| 1439 | 3 | 5 | next unless $csrc; | |||
| 1440 | 3 3 | 2 4 | my $crecs = eval { $csrc->fetch_all } // []; | |||
| 1441 | 3 | 4 | next if $@; | |||
| 1442 | 3 | 5 | my @ccols = _get_columns($csrc, $crecs); | |||
| 1443 | 3 | 3 | push @sources, [$crecs, \@ccols]; | |||
| 1444 | 3 | 11 | push @summaries, { label => $clabel }; | |||
| 1445 | } | |||||
| 1446 | ||||||
| 1447 | 3 | 147 | my ($records, $cols_ref) = @sources > 1 | |||
| 1448 | ? _combine_tables(\@sources) | |||||
| 1449 | : ($left_recs, \@left_cols); | |||||
| 1450 | 3 | 5 | my @columns = @$cols_ref; | |||
| 1451 | ||||||
| 1452 | 3 | 9 | my ($filtered, $filter_specs, $filters_json) = $self->_apply_filters($records); | |||
| 1453 | ||||||
| 1454 | 3 | 3 | my $title = $left_label; | |||
| 1455 | 3 3 | 5 19 | $title .= ' + ' . join(' + ', map { $_->{label} } @summaries) if @summaries; | |||
| 1456 | 3 | 5 | my $table_key = 'combine:' . lc($left_label); | |||
| 1457 | 3 | 7 | $table_key .= ':' . lc($_->{label}) for @summaries; | |||
| 1458 | ||||||
| 1459 | 3 | 8 | $self->render( | |||
| 1460 | template => "$platform/$language/dashboard", | |||||
| 1461 | handler => 'tt', | |||||
| 1462 | format => 'html', | |||||
| 1463 | records => $filtered, | |||||
| 1464 | columns => \@columns, | |||||
| 1465 | table => $table_key, | |||||
| 1466 | title => $title, | |||||
| 1467 | back_url => $self->_spec_to_url($left_spec), | |||||
| 1468 | back_label => "Back to $left_label", | |||||
| 1469 | left_spec => $left_spec, | |||||
| 1470 | combine_specs => \@combine_specs, | |||||
| 1471 | current_joins => [], | |||||
| 1472 | available_tables => $self->_scan_data_dir, | |||||
| 1473 | join_summaries => \@summaries, | |||||
| 1474 | filter_specs => $filter_specs, | |||||
| 1475 | filters_json => $filters_json, | |||||
| 1476 | export_url => $self->_build_export_url($left_spec, [], $filter_specs, \@combine_specs), | |||||
| 1477 | ); | |||||
| 1478 | } | |||||
| 1479 | ||||||
| 1480 - 1534 | =head2 export_data
C<GET /export> -- Stream the current logical view as a browser file download.
=head3 API SPECIFICATION
=head4 INPUT
?l= string (required) Left table spec.
?j= string (repeatable) Join steps.
?f= string (repeatable) Filter specs.
?format= string "csv" (default) or "sqlite".
=head4 DOMAIN CONSTRAINTS: ?format=
The comparison is C<$format eq 'sqlite'> -- case-sensitive, exact match.
=over 4
=item Valid partitions
C<csv> (explicit CSV), C<sqlite> (exact lowercase, SQLite binary),
absent/undef (defaults to CSV).
=item Invalid partitions (all fall back to CSV)
C<SQLITE> (uppercase, not equal to C<'sqlite'>), C<Sqlite> (mixed
case), C<sqlit> (truncated), C<sqlite1> (extra character), C<json>
(unknown format).
=back
=head4 OUTPUT
200 text/csv or application/vnd.sqlite3
404 application/json { "error": "Table not found" }
=head3 MESSAGES
None beyond the 404 response.
=head3 FORMAL SPECIFICATION
export_data == lambda self .
let (recs, cols, label) = _run_export_pipeline() in
pre recs /= undef
post if format = 'sqlite' then render_sqlite(recs, cols)
else render_csv(recs, cols)
=head3 EXAMPLE
GET /export?l=table:sales&format=csv -> downloads sales.csv
GET /export?l=table:sales&format=sqlite -> downloads sales.db
=cut | |||||
| 1535 | ||||||
| 1536 | 39 39 39 | 272186 43 32 | sub export_data ($self) { | |||
| 1537 | 39 | 80 | my ($records, $columns, $left_label) = $self->_run_export_pipeline; | |||
| 1538 | 39 | 1780 | return $self->reply->not_found unless $records; | |||
| 1539 | ||||||
| 1540 | 33 | 73 | my $format = $self->param('format') // 'csv'; | |||
| 1541 | 33 | 826 | $format = 'csv' unless $format eq 'sqlite'; | |||
| 1542 | 33 | 79 | (my $safe_name = lc $left_label) =~ s/[^a-z0-9_]+/_/g; | |||
| 1543 | ||||||
| 1544 | 33 | 99 | return $format eq 'sqlite' | |||
| 1545 | ? $self->_render_sqlite($records, $columns, $safe_name) | |||||
| 1546 | : $self->_render_csv($records, $columns, $safe_name); | |||||
| 1547 | } | |||||
| 1548 | ||||||
| 1549 - 1616 | =head2 export_write
C<POST /export> -- Write the current logical view to a chosen filesystem path.
=head3 API SPECIFICATION
=head4 INPUT
l= string (required) Left table spec.
j= string (repeatable) Join steps.
f= string (repeatable) Filter specs.
dir= string Target directory (must exist; resolved via realpath).
filename= string Output filename including extension. Extension determines
format: C<.csv> -> RFC 4180 CSV; C<.sql> -> SQLite.
=head4 DOMAIN CONSTRAINTS: filename=
The extension check uses C</\.csv\z/i> (CSV) or C</\.sql\z/i> (SQLite):
the C</i> flag makes matching case-insensitive, so C<.CSV> and C<.SQL>
are accepted alongside lowercase forms. Everything else returns 415.
Path separator characters (C</> and C<\>) in C<filename> are stripped
first via C<m{([^/\\]+)\z}> -- only the basename is kept, preventing
directory traversal.
=over 4
=item Valid partitions
C<report.csv>, C<report.sql>, C<REPORT.CSV>, C<report.SQL>. A
single-character stem (C<a.csv>) is also valid.
=item Invalid partitions (415)
C<report.txt>, C<report.json>, C<report> (no extension), empty string.
=back
=head4 OUTPUT
200 application/json { "saved": "/abs/path/to/file" }
404 application/json { "error": "..." } -- bad dir or table not found
415 application/json { "error": "..." } -- unsupported extension
500 application/json { "error": "..." } -- write failure
=head3 MESSAGES
error_dir_not_found -- realpath of dir failed or result is not a directory.
error_ext_required -- filename extension is not .csv or .sql.
error_write_failed -- DBI or filesystem write threw.
=head3 FORMAL SPECIFICATION
export_write == lambda self .
let dir = realpath(param('dir')) in
let file = dir / strip_path(param('filename')) in
pre is_dir(dir) /\ ext(file) in {csv, sql}
let (recs, cols, _) = _run_export_pipeline() in
pre recs /= undef
post spurt(file, serialise(recs, cols))
/\ render_json({ saved: file.to_string })
=head3 EXAMPLE
POST /export l=table:sales dir=/home/user/exports filename=report.csv
-> { "saved": "/home/user/exports/report.csv" }
=cut | |||||
| 1617 | ||||||
| 1618 | 38 38 38 | 207989 41 26 | sub export_write ($self) { | |||
| 1619 | 38 | 68 | my $dir = $self->param('dir') // ''; | |||
| 1620 | 38 | 6562 | my $filename = $self->param('filename') // ''; | |||
| 1621 | ||||||
| 1622 | 38 38 | 904 100 | my $dest_dir = eval { Mojo::File->new($dir)->realpath }; | |||
| 1623 | 38 | 1942 | unless (defined $dest_dir && -d $dest_dir) { | |||
| 1624 | 5 | 21 | return $self->render( | |||
| 1625 | json => { error => $self->_i18n('error_dir_not_found', $dir) }, | |||||
| 1626 | status => 404, | |||||
| 1627 | ); | |||||
| 1628 | } | |||||
| 1629 | ||||||
| 1630 | # Strip any path separators the browser might include. | |||||
| 1631 | 33 | 257 | ($filename) = $filename =~ m{([^/\\]+)\z}; | |||
| 1632 | 33 | 30 | my $format; | |||
| 1633 | 33 19 | 169 25 | if ($filename && $filename =~ /\.csv\z/i) { $format = 'csv'; } | |||
| 1634 | 4 | 5 | elsif ($filename && $filename =~ /\.sql\z/i) { $format = 'sqlite'; } | |||
| 1635 | else { | |||||
| 1636 | 10 | 22 | return $self->render( | |||
| 1637 | json => { error => $self->_i18n('error_ext_required') }, | |||||
| 1638 | status => 415, | |||||
| 1639 | ); | |||||
| 1640 | } | |||||
| 1641 | ||||||
| 1642 | 23 | 45 | my ($records, $columns, $left_label) = $self->_run_export_pipeline; | |||
| 1643 | 23 | 1129 | return $self->render(json => { error => 'Table not found' }, status => 404) | |||
| 1644 | unless $records; | |||||
| 1645 | ||||||
| 1646 | 21 | 52 | my $dest = $dest_dir->child($filename); | |||
| 1647 | 21 | 211 | eval { | |||
| 1648 | 21 | 38 | if ($format eq 'csv') { | |||
| 1649 | 17 | 30 | $dest->spurt(_serialize_csv($records, $columns)); | |||
| 1650 | } | |||||
| 1651 | else { | |||||
| 1652 | 4 | 10 | $dest->spurt($self->_write_sqlite_db($records, $columns)); | |||
| 1653 | } | |||||
| 1654 | }; | |||||
| 1655 | 21 | 1774 | return $self->render( | |||
| 1656 | json => { error => $self->_i18n('error_write_failed', $@) }, | |||||
| 1657 | status => 500, | |||||
| 1658 | ) if $@; | |||||
| 1659 | ||||||
| 1660 | 21 | 44 | $self->render(json => { saved => $dest->to_string }); | |||
| 1661 | } | |||||
| 1662 | ||||||
| 1663 - 1701 | =head2 dirs_api
C<GET /api/dirs> -- Return a JSON directory listing for the export panel.
=head3 API SPECIFICATION
=head4 INPUT
?path= string Directory to list (default: $HOME). Returns 404 when not
a directory. Hidden entries (names starting with ".") are
excluded.
=head4 OUTPUT
200 application/json
{
"path": "/abs/path",
"parent": "/abs/parent" or null (at filesystem root),
"dirs": [ { "name": "alpha", "path": "/abs/path/alpha" }, ... ]
}
404 application/json { "error": "Not a directory" }
=head3 MESSAGES
None beyond the 404 response.
=head3 FORMAL SPECIFICATION
dirs_api == lambda self .
let dir = realpath(param('path') // HOME) in
pre is_dir(dir)
post render_json({ path: dir, parent: parent(dir), dirs: subdirs(dir) })
=head3 EXAMPLE
GET /api/dirs?path=/home/user -> {"path":"/home/user","parent":"/home","dirs":[...]}
=cut | |||||
| 1702 | ||||||
| 1703 | 11 11 11 | 51576 12 6 | sub dirs_api ($self) { | |||
| 1704 | 11 | 19 | my $raw = $self->param('path') // $ENV{HOME} // '/'; | |||
| 1705 | 11 11 | 1290 32 | my $dir = eval { Mojo::File->new($raw)->realpath }; | |||
| 1706 | 11 | 832 | return $self->render(json => { error => 'Not a directory' }, status => 404) | |||
| 1707 | unless defined $dir && -d $dir; | |||||
| 1708 | ||||||
| 1709 | 6 | 46 | my $listing = $self->_list_dir($dir, 0); | |||
| 1710 | 6 | 15 | my $parent = $dir->dirname; | |||
| 1711 | ||||||
| 1712 | $self->render(json => { | |||||
| 1713 | path => $dir->to_string, | |||||
| 1714 | parent => ($parent->to_string ne $dir->to_string ? $parent->to_string : undef), | |||||
| 1715 | dirs => $listing->{dirs}, | |||||
| 1716 | 6 | 170 | }); | |||
| 1717 | } | |||||
| 1718 | ||||||
| 1719 - 1768 | =head2 stat_api
C<GET /api/stat> -- Return filesystem metadata for a file path.
Used by the home page tooltip to show modification time and size for recently
opened files without a page reload.
=head3 API SPECIFICATION
=head4 INPUT
?path= string Absolute path to query. Returns HTTP 400 when absent.
=head4 OUTPUT
200 application/json (file exists)
{ "exists": true, "path": "/resolved/path", "mtime": 1700000000, "size": 4096 }
200 application/json (file does not exist)
{ "exists": false, "path": "/original/path" }
400 application/json { "error": "\"path\" parameter is required" }
C<mtime> is Unix epoch seconds. C<size> is bytes. A missing/unresolvable
path returns HTTP 200 with C<exists: false> (not 404) so the UI can
distinguish "file was deleted" from a request error.
=head3 MESSAGES
error_path_required -- the "path" query parameter was not supplied.
=head3 FORMAL SPECIFICATION
stat_api == lambda self .
let path = param('path') in
pre path /= undef
let file = realpath(path) in
post if is_file(file) then
render_json({ exists: true, mtime: mtime(file), size: size(file) })
else render_json({ exists: false })
=head3 EXAMPLE
GET /api/stat?path=/data/sales.csv
-> { "exists": true, "path": "/data/sales.csv", "mtime": 1700000000, "size": 1234 }
GET /api/stat?path=/deleted.csv
-> { "exists": false, "path": "/deleted.csv" }
=cut | |||||
| 1769 | ||||||
| 1770 | 26 26 26 | 133921 30 23 | sub stat_api ($self) { | |||
| 1771 | 26 | 49 | my $path = $self->param('path'); | |||
| 1772 | 26 | 3036 | unless (defined $path && length $path) { | |||
| 1773 | 6 | 19 | return $self->render( | |||
| 1774 | json => { error => $self->_i18n('error_path_required') }, | |||||
| 1775 | status => 400, | |||||
| 1776 | ); | |||||
| 1777 | } | |||||
| 1778 | ||||||
| 1779 | 20 20 | 18 39 | my $file = eval { Mojo::File->new($path)->realpath }; | |||
| 1780 | # Restrict to files with a supported data extension so stat_api cannot be | |||||
| 1781 | # used as a filesystem oracle to probe /etc/shadow, /root/.ssh/, etc. | |||||
| 1782 | 20 | 1315 | return $self->render(json => { exists => \0, path => $path }) | |||
| 1783 | unless defined $file && -f $file && $file->basename =~ $EXT_RE; | |||||
| 1784 | ||||||
| 1785 | 11 | 418 | my @s = stat $file->to_string; | |||
| 1786 | 11 | 88 | $self->render(json => { | |||
| 1787 | exists => \1, | |||||
| 1788 | path => $file->to_string, | |||||
| 1789 | mtime => $s[9], | |||||
| 1790 | size => $s[7], | |||||
| 1791 | }); | |||||
| 1792 | } | |||||
| 1793 | ||||||
| 1794 - 1835 | =head2 upload_file
C<POST /upload> -- Accept a drag-and-dropped data file and return a redirect URL.
The file is saved under its original filename in a managed subdirectory of
C<< <app_home>/.uploads/ >>. The subdirectory name is randomised so that
concurrent uploads of files with the same name do not collide.
=head3 API SPECIFICATION
=head4 INPUT
Multipart form upload, field name: C<file>.
file upload Supported extensions: csv, db, sql, xml, psv.
=head4 OUTPUT
200 application/json { "url": "/open?path=/abs/path/file.csv", "path": "/abs/path/file.csv" }
400 application/json { "error": "No file received" }
415 application/json { "error": "Unsupported file type. Accepted: ..." }
=head3 MESSAGES
error_upload_none -- no file was received in the multipart upload.
error_upload_ext -- the file's extension is not in the supported list.
=head3 FORMAL SPECIFICATION
upload_file == lambda self .
let upload = req.upload('file') in
pre upload /= undef /\ basename(upload.filename) =~ EXT_RE
let dest = home/.uploads/<random>/<filename> in
post upload.move_to(dest)
/\ render_json({ url: '/open?path=' ++ url_escape(dest), path: dest })
=head3 EXAMPLE
POST /upload (multipart: file=@sales.csv)
-> { "url": "/open?path=%2F...%2Fsales.csv", "path": "/.../.uploads/abc123/sales.csv" }
=cut | |||||
| 1836 | ||||||
| 1837 | 29 29 29 | 201290 32 21 | sub upload_file ($self) { | |||
| 1838 | 29 | 42 | my $upload = $self->req->upload('file'); | |||
| 1839 | 29 | 2273 | unless ($upload && $upload->filename) { | |||
| 1840 | 4 | 10 | return $self->render( | |||
| 1841 | json => { error => $self->_i18n('error_upload_none') }, | |||||
| 1842 | status => 400, | |||||
| 1843 | ); | |||||
| 1844 | } | |||||
| 1845 | ||||||
| 1846 | # Enforce upload size limit. max_request_size in startup() sets the | |||||
| 1847 | # transport-layer cap, but Mojolicious still calls the controller when the | |||||
| 1848 | # limit fires (with req->is_limit_exceeded true and partial content in the | |||||
| 1849 | # upload asset). Check is_limit_exceeded first; fall through to an asset | |||||
| 1850 | # size check as a belt-and-braces guard for any bytes that slipped through. | |||||
| 1851 | 25 | 97 | if ($self->req->is_limit_exceeded || $upload->size > $MAX_UPLOAD_BYTES) { | |||
| 1852 | 3 | 23 | return $self->render( | |||
| 1853 | json => { error => $self->_i18n('error_upload_too_large', $MAX_UPLOAD_MIB) }, | |||||
| 1854 | status => 413, | |||||
| 1855 | ); | |||||
| 1856 | } | |||||
| 1857 | ||||||
| 1858 | # /s so .* matches \n; a percent-decoded newline inside a browser filename | |||||
| 1859 | # must not silently truncate the directory-strip, leaving "dir\ncomponent" in. | |||||
| 1860 | 22 | 345 | (my $filename = $upload->filename) =~ s{.*[/\\]}{}s; | |||
| 1861 | 22 | 126 | unless ($filename && $filename =~ $EXT_RE) { | |||
| 1862 | 5 | 41 | return $self->render( | |||
| 1863 | json => { error => $self->_i18n('error_upload_ext') }, | |||||
| 1864 | status => 415, | |||||
| 1865 | ); | |||||
| 1866 | } | |||||
| 1867 | ||||||
| 1868 | # Store uploads in <app_home>/.uploads/<random_subdir>/<original_filename>. | |||||
| 1869 | # Using the original filename is essential: Database::Abstraction derives | |||||
| 1870 | # the table name from the file stem, so "sales.csv" must stay "sales.csv". | |||||
| 1871 | # The random subdirectory prevents collisions from concurrent same-name uploads. | |||||
| 1872 | 17 | 157 | my $uploads_base = $self->app->home->child('.uploads'); | |||
| 1873 | 17 | 279 | $uploads_base->make_path unless -d $uploads_base; | |||
| 1874 | 17 | 344 | my $sub_dir = tempdir(DIR => $uploads_base->to_string, CLEANUP => 0); | |||
| 1875 | 17 | 3389 | my $dest = Mojo::File->new($sub_dir)->child($filename)->to_string; | |||
| 1876 | 17 | 272 | $upload->move_to($dest); | |||
| 1877 | ||||||
| 1878 | 17 | 1208 | $self->render(json => { | |||
| 1879 | url => '/open?path=' . url_escape($dest), | |||||
| 1880 | path => $dest, | |||||
| 1881 | }); | |||||
| 1882 | } | |||||
| 1883 | ||||||
| 1884 | 1; | |||||
| 1885 | ||||||