lib/Geo/Coder/Free.pm — Duplicate Code Regions
◆ Amber lines duplicate code in another file — hover for location.
◆ Purple: duplicate of another region within this same file.
1: package Geo::Coder::Free;
2:
3: use strict;
4: use warnings;
5: use autodie qw(:all);
6:
7: use Carp;
8: use Geo::Coder::Free::Local;
9: use Geo::Coder::Free::MissCache;
10: use Geo::Coder::Free::MaxMind;
11: use Geo::Coder::Free::OpenAddresses;
12: use Geo::Coder::Free::Utils qw(_abbreviate _normalize);
13: use Geo::Location::Predicate qw(_valid_location);
14: use Object::Configure;
15: use Params::Get;
16: use Readonly;
17: use Scalar::Util;
18: # Imported functions are not methods: without this, $obj->croak() and the like
19: # would resolve. The three kept are reached as
20: # Geo::Coder::Free::<name> by t/function.t, t/path.t and t/edge_cases.t.
21: use namespace::clean -except => [qw(_valid_location _normalize _abbreviate)];
22:
23: # TODO: from the 0.50 gap analysis
24: #
25: # Errors and housekeeping:
26: # TODO: OpenAddresses.pm croaks with the raw address as its whole message
27: # (Carp::croak($city), "TODO: do something here"), and croaks when
28: # Geo::StreetAddress::US finds no state. Neither is documented or goes
29: # through a message table; return undef instead, or document them
30: # TODO: MaxMind->new() croaks ": directory ... doesn't exist" with no package
31: # name, because ref($class) is '' when called on the class
32: # TODO: xt/mutant_*.t stubs are committed by CI with 117 TODOs, against
33: # their own "DO NOT COMMIT" header; complete them or stop committing them
34: #
35: # Coverage and testing:
36: # TODO: ship a small fixture openaddresses.sql and cities.sql so t/openaddr.t
37: # and t/maxmind.t run in CI. CI branch coverage is 10.6% for
38: # OpenAddresses.pm and 26.9% for MaxMind.pm because neither database is there
39: # TODO: accuracy tests - a table of well-known cities checked to within a
40: # tolerance with Utils::distance(), which would have caught the
41: # "City, Country" bug below
42: # TODO: make sure .github/workflows/bare.yml (optional modules hidden) runs on
43: # every push; it is the only job that catches optional-module crashes such
44: # as 0.49's Geo::Address::Parser one
45: #
46: # Accuracy:
47: # TODO: two-part "City, Country" in MaxMind.pm answers with the first row in a
48: # region ("Berlin, Germany" and "Munich, Germany" both give 52.08,7.38;
49: # "Toronto, Canada" lands in British Columbia; "Paris, France" is not
50: # found). Choose the most populous matching city in the country - cities.sql
51: # has Population - which also covers the TODO about returning the central
52: # location in MaxMind.pm's geocode POD
53: # TODO: fold accents (e.g. Text::Unidecode) as well as matching exactly, so
54: # "Zurich" finds the accented spelling in the data
55: #
56: # Features:
57: # TODO: return ranked candidates with confidence in list context, rather than
58: # one point (the "Portlands" TODO in OpenAddresses.pm's geocode POD)
59: # TODO: move %known_locations (MaxMind.pm, OpenAddresses.pm) and Local.pm's
60: # %alternatives into data files, and add the "dbname" parameter to
61: # OpenAddresses->new() that its BUGS section asks for
62: # TODO: Canadian and Australian address parsing (Local.pm), and non-US parsing
63: # in OpenAddresses.pm, for users without Geo::libpostal
64: # TODO: JSON/XML output from the website (Display/query.pm)
65: #
66: # Technical debt:
67: # TODO: break OpenAddresses::geocode() (about 900 lines, US parsing nested
68: # eight deep) into per-country handlers that can be tested on their own
69: # TODO: Free.pm, Local.pm and MaxMind.pm new() "only work with no arguments"
70: # when called as a function (FIXME in each)
71: # TODO: resolve the TODO/FIXME markers in the VWF-owned Allow.pm, Display.pm,
72: # Utils.pm and Config.pm, in ../vwf, and sync
73:
74: =head1 NAME
75:
76: Geo::Coder::Free - Geocoding using free, locally-hosted databases
77:
78: =head1 VERSION
79:
80: Version 0.51
81:
82: =cut
83:
84: our $VERSION = '0.51';
85:
86: =encoding utf-8
87:
88: =head1 SYNOPSIS
89:
90: use Geo::Coder::Free;
91:
92: my $geo = Geo::Coder::Free->new();
93: my $pt = $geo->geocode(location => 'Ramsgate, Kent, UK');
94: printf "%.6f, %.6f\n", $pt->lat(), $pt->long();
95:
96: # With OpenAddresses/WhoOnFirst data:
97: my $geo2 = Geo::Coder::Free->new(openaddr => $ENV{OPENADDR_HOME});
98: my $pt2 = $geo2->geocode(location => '1600 Pennsylvania Avenue NW, Washington DC, USA');
99:
100: # Free-text scanning:
101: my @hits = $geo2->geocode(scantext => 'She grew up in Ramsgate, Kent.',
102: region => 'GB');
103:
104: =head1 DESCRIPTION
105:
106: C<Geo::Coder::Free> translates addresses into latitude/longitude coordinates
107: using local SQLite databases built from free data sources - MaxMind/GeoNames,
108: OpenAddresses, Who's On First, OpenStreetMap, and dr5hn's countries/states/cities
109: database. It deliberately avoids paid or rate-limited online geocoding services.
110: The module is designed to be flexible, supporting both command-line and programmatic usage.
111: It also includes a sample CGI script for a web-based geocoding service.
112:
113: Geocoding dispatch order depends on whether C<OPENADDR_HOME> (or C<openaddr>) is set:
114:
115: B<With OpenAddresses data:>
116:
117: =over 4
118:
119: =item 1. C<Geo::Coder::Free::OpenAddresses> - requires C<OPENADDR_HOME>
120:
121: =item 2. C<Geo::Coder::Free::Local> - user-curated CSV entries (tried as fallback)
122:
123: =item 3. C<Geo::Coder::Free::MaxMind> - bundled, always available
124:
125: =back
126:
127: B<Without OpenAddresses data:>
128:
129: =over 4
130:
131: =item 1. C<Geo::Coder::Free::MaxMind> only - Local is not consulted.
132:
133: =back
134:
135: The C<cgi-bin> directory contains a simple DIY geo-coding website:
136:
137: cgi-bin/page.fcgi page=query q=1600+Pennsylvania+Avenue+NW+Washington+DC+USA
138:
139: The sample website is currently down while a new host is sought.
140: When it returns, you will be able to test it with:
141:
142: curl 'https://geocode.nigelhorne.com/cgi-bin/page.fcgi?page=query&q=1600+Pennsylvania+Avenue+NW+Washington+DC+USA'
143:
144: =head1 LIMITATIONS
145:
146: =over 4
147:
148: =item * C<scantext> mode only finds locations in OpenAddresses; it falls back
149: silently when C<OPENADDR_HOME> is not set (B<FIXME>: should warn).
150:
151: =item * The C<__DATA__> alternatives table is hard-coded; it should live in a
152: user-editable config file.
153:
154: =item * The address-regex scantext path misses birth-year sentences such as
155: C<"She was born May 21, 1937 in Noblesville, IN."> because the regex requires
156: a preceding capital-letter word directly before the city.
157:
158: =item * C<reverse_geocode> is only partially implemented; the MaxMind path does
159: not return meaningful results.
160:
161: =item * Without OpenAddresses data, a two-part C<"City, Country"> location is
162: often answered wrongly. The MaxMind backend reads the first part as a region
163: when it can, and answers with the first row it finds in that region, not with
164: the city: C<"Berlin, Germany"> and C<"Munich, Germany"> both come back at
165: 52.08, 7.38, and C<"Toronto, Canada"> comes back in British Columbia. Some
166: others, such as C<"Paris, France">, are not found at all. These results carry
167: C<< accuracy => 'region' >> and C<< confidence => 0.1 >>, so a caller that needs
168: a city should check L</provenance> and reject them. Naming the region as well,
169: as in C<"Munich, Bavaria, Germany">, gives the right answer.
170:
171: =item * Place names are matched as they are spelled in the data, with no
172: folding of accents: C<"ZE<uuml>rich, Switzerland"> is found and C<"Zurich, Switzerland">
173: is not. Pass locations as Perl character strings (decoded, for example by
174: C<use utf8> or an C<:encoding(UTF-8)> layer), not as undecoded UTF-8 bytes,
175: since a byte string does not match the name in the data.
176:
177: =back
178:
179: =cut
180:
181: # -----------------------------------------------------------------------
182: # Module-level singletons - initialised once, shared across all instances.
183: # Using 'our' so that test code can reset them between test runs if needed.
184: # -----------------------------------------------------------------------
185: our $alternatives;
186:
187: # The alternatives being tried by geocode() calls further up the stack. A key
188: # is skipped while it is in here, so an entry whose replacement still matches
189: # it (or two entries that rewrite into each other) cannot recurse forever.
190: my %_alternatives_active;
191:
192: # -----------------------------------------------------------------------
193: # Error-message table. All user-facing strings live here so that a future
194: # i18n layer only needs to swap this hash, not touch every call site.
195: # -----------------------------------------------------------------------
196: my %_MESSAGES = (
197: usage_geocode => 'Usage: %s::geocode(location => $location|scantext => $text)',
198: usage_geocode_many => 'Usage: %s::geocode_many(\@values, region => $region, on_error => "croak"|"undef")',
199: usage_reverse => 'Usage: %s::reverse_geocode(latlng => "$lat,$long")',
200: invalid_location => '%s: invalid location to geocode(), %s',
201: invalid_scantext => '%s: invalid scantext to geocode(), %s',
202: invalid_on_error => '%s: on_error must be "croak" or "undef", not %s',
203: reserved_location => '%s: "%s" is one of the keys geocode_many() answers with, so it cannot be one of the values it looks up',
204: geocoding_failed => '%s: geocoding failed',
205: reverse_unsupported => 'Reverse lookup is not yet supported',
206: use_arrow_new => '%s: use ->new() not ::new() to instantiate',
207: bad_ref_arg => 'Usage: %s — do not pass a non-hash reference',
208: );
209:
210: # Confidence thresholds for scantext results, expressed as named constants
211: # rather than magic numbers so call sites document intent.
212: Readonly::Scalar my $CONF_TRIPLET => 0.8;
213: Readonly::Scalar my $CONF_DUPLET => 0.7;
214: Readonly::Scalar my $CONF_REGEX => 0.7;
215:
216: # The keys geocode_many() answers with besides the values themselves. The map
217: # and the counts share one hashref, so a value spelled like one of these would
218: # overwrite what the name stands for, and it is refused by name instead
219: Readonly::Array my @RESULT_KEYS => qw(lookups reads resolved unresolved unresolved_values);
220:
221: # The backends geocode_many() reports its reads for: the slots of the facade
222: # that hold them, and the name each is reported under. The names are the short
223: # vocabulary provenance() reports a result's backend in, so that the backend
224: # that read a value and the backend a result came from are spelled the same way
225: # and a caller can read the two together.
226: Readonly::Hash my %_BACKEND_NAMES => (
227: maxmind => 'MaxMind',
228: local => 'Local',
229: openaddr => 'OpenAddresses',
230: );
231:
232: # Stopwords excluded from word-window scantext searches.
233: my %_COMMON_WORDS = map { $_ => 1 } qw(
234: a age an and at be by cross for how i in is more of on or over pm
235: road she side some the to was with
236: );
237:
238: =head1 METHODS
239:
240: =head2 new
241:
242: =head3 SYNOPSIS
243:
244: my $geo = Geo::Coder::Free->new();
245: my $geo = Geo::Coder::Free->new(openaddr => '/data/openaddr');
246: my $geo = Geo::Coder::Free->new(directory => '/data/maxmind');
247:
248: =head3 DESCRIPTION
249:
250: Constructor. Accepts a hash or hashref of options. If called without
251: C<openaddr>, the module checks C<$ENV{OPENADDR_HOME}> before giving up.
252:
253: If called on an existing object instance (C<< $clone = $geo->new() >>), returns
254: a B<shallow clone>. All scalar fields are copied by value, but reference-type
255: fields (C<alternatives>, C<scantext_misses>, C<maxmind>, C<openaddr>) share the
256: same underlying object or hashref between the original and the clone. Mutations
257: to those shared references are immediately visible in both objects.
258:
259: =head3 API SPECIFICATION
260:
261: =head4 input
262:
263: # Input schema (Params::Validate::Strict)
264: openaddr => { type => 'scalar', optional => 1 } # path to OpenAddresses/WOF data dir
265: directory => { type => 'scalar', optional => 1 } # path to MaxMind/GeoNames files
266: cache => { type => 'object', optional => 1, can => ['get', 'set'] } # CHI-compatible cache object
267: libpostal => { type => 'boolean', optional => 1, default => 1 } # false: OpenAddresses never uses Geo::libpostal
268:
269: C<libpostal> is passed to L<Geo::Coder::Free::OpenAddresses>; see its
270: "Optional modules" section for why you might turn it off.
271:
272: =head4 output
273:
274: # Output schema (Return::Set)
275: { type => 'object', isa => 'Geo::Coder::Free' }
276:
277: =head3 EXAMPLE
278:
279: use Geo::Coder::Free;
280:
281: # Minimal - uses only the bundled MaxMind data:
282: my $geo = Geo::Coder::Free->new();
283:
284: # Full - also searches OpenAddresses/WOF:
285: my $geo = Geo::Coder::Free->new(openaddr => $ENV{OPENADDR_HOME});
286:
287: =head3 MESSAGES
288:
289: use ->new() not ::new() Called as a function; use arrow syntax.
290:
291: =cut
292:
293: sub new {
294: my $class = shift;
295:
296: my $params = Params::Get::get_params(undef, @_) // {};
297:
298: # Called as a function (Geo::Coder::Free::new) rather than a method
299: if (!defined($class)) {
300: if (keys %{$params}) {
301: ◆ Carp::carp(_i18n('use_arrow_new', __PACKAGE__)); 302: ◆ return; 303: ◆ } 304: ◆ $class = __PACKAGE__; # FIXME: only works with no arguments 305: ◆ } elsif (Scalar::Util::blessed($class)) { 306: ◆ return bless { %{$class}, %{$params} }, ref($class); 307: ◆ } 308: ◆ 309: ◆ # Populate the alternatives map once from __DATA__ and cache it 310: ◆ # across all instances. Each line is "from = to"; the text either side 311: # is kept as written (Config::Auto, used before, split a value into words,
312: # so "Borough of North Tyneside" became "Borough, of, North, Tyneside").
313: if (!$alternatives) {
314: $alternatives = {};
315: while (my $line = <DATA>) {
316: next if $line =~ /^\s*(#|$)/;
317: my ($from, $to) = split /\s*=\s*/, $line, 2;
318: next unless defined($to);
319: s/^\s+|\s+$//g for $from, $to;
320: $alternatives->{$from} = $to if length($from) && length($to);
321: }
322: }
323:
324: # Resolve OPENADDR_HOME before Object::Configure so plug-ins can see it
325: if (!defined($params->{'openaddr'}) && $ENV{'OPENADDR_HOME'}) {
326: $params->{'openaddr'} = $ENV{'OPENADDR_HOME'};
327: }
328:
329: $params = Object::Configure::configure($class, $params);
330:
331: my $rc = {
332: %{$params},
333: maxmind => Geo::Coder::Free::MaxMind->new($params),
334: alternatives => $alternatives,
335: };
336:
337: if ($params->{'openaddr'}) {
338: $rc->{'openaddr'} = Geo::Coder::Free::OpenAddresses->new(id => 'md5', %{$params});
339: }
340: if (my $cache = $params->{'cache'}) {
341: $rc->{'cache'} = $cache;
342: }
343:
344: return bless $rc, $class;
345: }
346:
347: =head2 valid_location
348:
349: =head3 SYNOPSIS
350:
351: if ($geo->valid_location($row->{'place'})) {
352: push @found, $geo->geocode(location => $row->{'place'});
353: }
354:
355: =head3 DESCRIPTION
356:
357: Answers whether a string is something this module will even try to look up.
358: It is the public form of the rule C<geocode()> applies before it touches a
359: backend, so a caller working down a spreadsheet column can skip a row whose
360: value is a bare number - or is empty - instead of catching the croak that
361: would otherwise kill the whole run. That rule is defined once, in
362: L<Geo::Location::Predicate>, and the backends apply it too: what
363: C<valid_location()> refuses is refused wherever it is looked up.
364:
365: Only that cheap syntactic test is answered. A true result means the string is
366: not disqualified, B<not> that a backend knows it: C<'Ramsgate, Kent'> is valid
367: and may still find nothing, while C<'0044'> is invalid and is never looked up.
368: The rule rejects a string of digits only, which is what a numeric column holds
369: when the wrong column has been read, so anything with a non-digit in it counts
370: as valid - C<'3.14'> and C<'-'> included. C<undef>, C<''> and a reference are
371: not valid.
372:
373: =head3 API SPECIFICATION
374:
375: =head4 input
376:
377: location => { type => 'scalar', optional => 1 } # the string to test
378:
379: =head4 output
380:
381: { type => 'boolean' } # false for undef, '' and a digit-only string
382:
383: =head3 EXAMPLE
384:
385: my $geo = Geo::Coder::Free->new();
386:
387: $geo->valid_location('Ramsgate, Kent'); # true
388: $geo->valid_location('0044'); # false - never a location
389: $geo->valid_location(''); # false
390: $geo->valid_location(undef); # false
391:
392: Accepts the three calling conventions the rest of this module does:
393:
394: $geo->valid_location('0044');
395: Geo::Coder::Free->valid_location('0044');
396: Geo::Coder::Free::valid_location('0044');
397:
398: =cut
399:
400: sub valid_location {
401: my $self = shift;
402:
403: # The location is whichever argument is not the invocant, so that the
404: # predicate reads the same whether it is called on an instance, on the
405: # class or as a plain function.
406: my $location;
407: if (ref($self)) {
408: $location = shift;
409: } elsif (@_) {
410: $location = shift;
411: } elsif (defined($self) && $self ne __PACKAGE__) {
412: $location = $self;
413: }
414:
415: return _valid_location($location) ? 1 : 0;
416: }
417:
418: =head2 geocode
419:
420: =head3 SYNOPSIS
421:
422: # Standard lookup (returns a Geo::Location::Point or undef)
423: my $pt = $geo->geocode(location => 'Ramsgate, Kent, UK');
424: printf "lat=%.6f lon=%.6f\n", $pt->lat(), $pt->long();
425:
426: # Scantext - returns a list of Geo::Location::Point objects
427: my @hits = $geo->geocode(
428: scantext => 'She lived in Ramsgate, Kent.',
429: region => 'GB',
430: ignore_words => [qw(lived)],
431: );
432:
433: # Invocation flexibility (all equivalent)
434: $geo->geocode('Ramsgate, Kent, UK');
435: $geo->geocode({ location => 'Ramsgate, Kent, UK' });
436: $geo->geocode(location => 'Ramsgate, Kent, UK');
437:
438: # A batch caller with a dirty column: skip the row, do not kill the run
439: for my $place (@column) {
440: my $pt = $geo->geocode(location => $place, on_error => 'undef');
441: next unless defined $pt;
442: push @found, $pt;
443: }
444:
445: =head3 DESCRIPTION
446:
447: A location or a scantext that is not a location at all - a bare number, such as
448: the C<0044> a numeric column holds - is refused before any backend is
449: touched. C<valid_location()> answers the same question, so a caller can filter
450: first; C<on_error> chooses what happens when one gets through anyway:
451:
452: =over 4
453:
454: =item * C<'croak'> (the default) - C<Carp::croak>, the behaviour every existing
455: caller already has.
456:
457: =item * C<'undef'> - return C<undef>, so one bad row in a spreadsheet column is
458: skipped rather than ending the run. It changes the answer for B<only> the two
459: documented input errors below; an exception from a backend is still an
460: exception, and a malformed argument list still croaks.
461:
462: =back
463:
464: The option is deliberately narrow, so a batch caller wants C<valid_location()>
465: with it: that predicate answers false for C<undef> and C<''> as well, which are
466: the other two things an empty cell holds. A call with neither a location nor a
467: scantext is a usage error rather than an input error - the same croak as calling
468: C<geocode()> with no arguments - and stays a croak on either setting.
469:
470: =head3 API SPECIFICATION
471:
472: =head4 input
473:
474: # Input schema (Params::Validate::Strict) - exactly one of location or scantext is required
475: location => { type => 'scalar', optional => 1 } # address string (exclusive with scantext)
476: scantext => { type => 'scalar', optional => 1 } # free text to scan for place names
477: region => { type => 'scalar', optional => 1 } # ISO 3166-1 alpha-2 country code hint
478: ignore_words => { type => 'arrayref', optional => 1 } # words to suppress during scantext scan
479: on_error => { type => 'scalar', optional => 1 } # 'croak' (default) or 'undef'
480:
481: =head4 output
482:
483: # Output schema (Return::Set)
484: # scalar context: { type => 'object', isa => 'Geo::Location::Point', optional => 1 }
485: # list context: { type => 'arrayref', of => { isa => 'Geo::Location::Point' } }
486:
487: =head4 provenance
488:
489: Every result carries C<geocoder>, C<database>, C<confidence> and C<accuracy>
490: which distribution produced
491: it, which backend answered, how good the match was, and how precise the
492: coordinate is. C<provenance()> reads the four back. C<location> is set by every
493: path but the MaxMind cities one, and C<text> only by a scantext lookup; C<lat>,
494: C<long> and C<lng> come from the value object itself.
495:
496: =head3 MESSAGES
497:
498: Usage: ...::geocode(...) No location or scantext argument given.
499: invalid location to geocode() location is purely numeric.
500: invalid scantext to geocode() scantext is purely numeric.
501: on_error must be ... on_error is neither 'croak' nor 'undef'.
502:
503: =head3 PSEUDOCODE
504:
505: if self is not a blessed object -> delegate to new()->geocode(@args)
506: normalise @_ into %params
507: validate: on_error is 'croak' or 'undef' (croak if not)
508: validate: location is not purely numeric; scantext is not purely numeric
509: (undef instead of croaking when on_error is 'undef')
510: if openaddr backend is available:
511: if scantext:
512: try the raw scantext string as a direct location
513: build stopword set from %_COMMON_WORDS + ignore_words param
514: try 3-word windows (triplets) at confidence 0.8
515: try 2-word windows (duplets) at confidence 0.7
516: try the address-pattern regex at confidence 0.7
517: try region-specific address finders (GB / US / CA)
518: mark scantext as a miss; return undef
519: else:
520: try openaddr backend
521: try local backend
522: try __DATA__ alternatives map
523: try maxmind backend for location lookups
524: croak if no scantext and no location
525:
526: =cut
527:
528: sub geocode {
529: my $self = shift;
530:
531: # Handle being called as a function rather than a method
532: if (!ref($self)) {
533: if (scalar @_) {
534: return __PACKAGE__->new()->geocode(@_);
535: } elsif (!defined($self)) {
536: Carp::croak(_i18n('usage_geocode', __PACKAGE__));
537: } elsif ($self eq __PACKAGE__) {
538: Carp::croak(_i18n('usage_geocode', $self));
539: }
540: return __PACKAGE__->new()->geocode($self);
541: } elsif (ref($self) eq 'HASH') {
542: return __PACKAGE__->new()->geocode($self);
543: }
544:
545: my %params = _normalize_args(_i18n('usage_geocode', __PACKAGE__), 'location', @_);
546:
547: # What to do with an input this module rejects before it touches a backend.
548: # 'croak' is the historical behaviour and stays the default, so no existing
549: # caller changes; 'undef' is for a batch caller working down a dirty column,
550: # which wants the row skipped rather than the whole run killed. Only the two
551: # input errors below consult it: a backend that throws still throws, because
552: # nothing here is wrapped in an eval
553: my $on_error = $params{'on_error'} // 'croak';
554: Carp::croak(_i18n('invalid_on_error', __PACKAGE__, $on_error))
555: unless $on_error eq 'croak' || $on_error eq 'undef';
556:
557: # Reject pure-numeric inputs early - they are never valid addresses.
558: # The rule is Geo::Location::Predicate's, the same one the backends
559: # apply, and valid_location() answers it in public, so a caller can
560: # pre-filter with it and never reach this croak at all.
561: if (defined($params{'location'}) && !_valid_location($params{'location'})) {
562: return if !length($params{'location'}) || ($on_error eq 'undef');
563: Carp::croak(_i18n('invalid_location', __PACKAGE__, $params{'location'}));
564: }
565: if (defined($params{'scantext'}) && !_valid_location($params{'scantext'})) {
566: return if !length($params{'scantext'}) || ($on_error eq 'undef');
567: Carp::croak(_i18n('invalid_scantext', __PACKAGE__, $params{'scantext'}));
568: }
569:
570: if ($self->{'openaddr'}) {
571: if (my $scantext = $params{'scantext'}) {
572: return if $self->{'scantext_misses'}{$scantext};
573:
574: # First try the whole scantext as a direct location lookup -
575: # saves work when the text happens to be a valid address string.
576: $self->{'local'} ||= Geo::Coder::Free::Local->new();
577: my @direct = grep { defined }
578: $self->{'local'}->geocode($scantext),
579: $self->{'openaddr'}->geocode($scantext),
580: $self->{'maxmind'}->geocode($scantext);
581: return @direct if @direct;
582:
583: my $region = $params{'region'};
584:
585: # Build the effective stopword set. When the caller supplies no
586: # ignore_words we avoid copying %_COMMON_WORDS (which is immutable
587: # at runtime) by using a reference to it directly. Only when extra
588: # words are needed do we allocate and populate a merged hash.
589: my $ignore_words;
590: if (my $iw = $params{'ignore_words'}) {
591: my %merged = %_COMMON_WORDS;
592: $merged{lc $_} = 1 for @{$iw};
593: $ignore_words = \%merged;
594: } else {
595: $ignore_words = \%_COMMON_WORDS;
596: }
597:
598: # 3-word window search
599: my @triplets = _find_word_ngrams($scantext, 3, $ignore_words);
600: my $res = $self->_resolve_scan_candidates(\@triplets, $region, $CONF_TRIPLET, $scantext);
601: if (@{$res}) {
602: return wantarray ? @{$res} : $res->[0];
603: }
604:
605: # 2-word window search
606: my @duplets = _find_word_ngrams($scantext, 2, $ignore_words);
607: $res = $self->_resolve_scan_candidates(\@duplets, $region, $CONF_DUPLET, $scantext);
608: if (@{$res}) {
609: return wantarray ? @{$res} : $res->[0];
610: }
611:
612: # Regex-based address pattern - catches "City, ST"-style fragments.
613: # Note: misses sentences like "born May 21, 1937 in Noblesville, IN"
614: # because the pattern requires a capitalised word before the city.
615: my @regex_matches = $scantext =~
616: /\b(?:\d+\s+)?(?:[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\.?),\s*
617: (?:[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*(?:,\s*[A-Z]{2,})*)\b/gx;
618: my @places = grep { defined && $_ ne '' } @regex_matches;
619: $res = $self->_resolve_scan_candidates(\@places, $region, $CONF_REGEX, $scantext);
620: if (@{$res}) {
621: return wantarray ? @{$res} : $res->[0];
622: }
623:
624: # Region-specific structured address patterns
625: if ($region) {
626: my @candidates;
627: if ($region eq 'GB') { @candidates = _find_gb_addresses($scantext) }
628: elsif ($region eq 'US') { @candidates = _find_us_addresses($scantext) }
629: elsif ($region eq 'Canada') { @candidates = _find_ca_addresses($scantext) }
630:
631: if (@candidates) {
632: my @regional;
633: for my $candidate (@candidates) {
634: next if $ignore_words->{lc $candidate};
635: my @hits = grep { defined }
636: $self->{'openaddr'}->geocode("$candidate, $region");
637: push @regional, @hits if @hits;
638: }
639: return @regional if @regional;
640: }
641: }
642:
643: $self->{'scantext_misses'}{$scantext} = 1;
644: return;
645: }
646:
647: # Standard (non-scantext) lookup path
648: if (wantarray) {
649: my @rc = $self->{'openaddr'}->geocode(\%params);
650: return @rc if @rc && $rc[0];
651: $self->{'local'} ||= Geo::Coder::Free::Local->new();
652: @rc = $self->{'local'}->geocode(\%params);
653: return @rc if @rc && $rc[0];
654: } else {
655: if (my $rc = $self->{'openaddr'}->geocode(\%params)) {
656: return $rc;
657: }
658: $self->{'local'} ||= Geo::Coder::Free::Local->new();
659: if (my $rc = $self->{'local'}->geocode(\%params)) {
660: return $rc;
661: }
662: }
663:
664: # Try the alternatives table - hand-curated mappings for locations that
665: # the databases have under slightly different names.
666: # M3 (transitive reduction): every scantext path inside this block returned
667: # early above; at this point $params{'scantext'} is provably falsy, so the
668: # former `if (!$params{'scantext'})` guard was a tautology - removed.
669: if (my $alt = $self->{'alternatives'}) {
670: my $location = $params{'location'};
671: # Not each(): the recursive geocode() calls below walk the same
672: # hash, sharing and resetting its iterator, so an each() loop here
673: # would start again from the top and retry the same key forever.
674: foreach my $key (sort keys %{$alt}) {
675: next if $_alternatives_active{$key};
676: my $value = $alt->{$key};
677: next unless $location =~ $key;
678: local $_alternatives_active{$key} = 1;
679: (my $new_loc = $location) =~ s/$key/$value/;
680: $params{'location'} = $new_loc;
681: if (my $rc = $self->geocode(\%params)) {
682: return $rc;
683: }
684: # Also try the comma-free variant ("Tyne and Wear" etc.)
685: if ($value =~ /, /) {
686: (my $flat = $value) =~ s/,//g;
687: ($new_loc = $location) =~ s/$key/$flat/;
688: $params{'location'} = $new_loc;
689: if (my $rc = $self->geocode(\%params)) {
690: return $rc;
691: }
692: }
693: # Restore for the next iteration
694: $params{'location'} = $location;
695: }
696: }
697: }
698:
699: # Final fallback: MaxMind for location lookups.
700: # Scantext without OPENADDR_HOME will silently reach here and return undef;
701: # a proper fix would warn here (see LIMITATIONS).
702: # M8 (tautology elimination): both arms of the former wantarray ternary were
703: # identical - collapsed to a single call; context propagates implicitly.
704: if ($params{'location'}) {
705: return $self->{'maxmind'}->geocode(\%params);
706: }
707:
708: Carp::croak(_i18n('usage_geocode', __PACKAGE__)) unless $params{'scantext'};
709: return;
710: }
711:
712: =head2 geocode_many
713:
714: =head3 SYNOPSIS
715:
716: my $batch = $geo->geocode_many(\@places, region => 'GB', on_error => 'undef');
717:
718: for my $place (@places) {
719: my $pt = $batch->{$place}; # a Geo::Location::Point, or undef
720: next unless defined $pt;
721: printf "%s is at %.4f, %.4f\n", $place, $pt->lat(), $pt->long();
722: }
723:
724: printf "geocoded %d of %d places in %d lookups\n",
725: $batch->{'resolved'},
726: $batch->{'resolved'} + $batch->{'unresolved'},
727: $batch->{'lookups'};
728:
729: =head3 DESCRIPTION
730:
731: The loop a batch caller writes around C<geocode()>, run for it: one lookup per
732: B<distinct> value, with the same options and the same results, plus the counts
733: that caller would otherwise keep itself - how many values made a backend read,
734: how many resolved, how many did not, which ones, and which backend read for them
735:
736: The hashref returned holds one entry per distinct value given, keyed by the
737: value, plus five keys of its own:
738:
739: =over 4
740:
741: =item * C<< $batch->{$value} >> - the C<Geo::Location::Point> the value resolved
742: to, or C<undef> if nothing did. Every distinct value in the input is keyed, so
743: a caller walking a column can look each row up without first asking whether the
744: batch covered it. An C<undef> in the input has no key of its own and shares the
745: entry of the empty string, which is not a location either.
746:
747: =item * C<lookups> - how many distinct values made a backend read to answer
748: them: one per value, however many backends looked at it. A value the module
749: refuses before any backend is touched - a bare number, C<''>, C<undef>, a
750: reference - is not counted, and neither is a value a backend answered from a
751: cache or from a hand-curated entry, because nothing was read for it. This is
752: what makes the count the work a caller is capping rather than the work it asked
753: for. A value repeated in the batch is one lookup. Every backend reports its
754: reads through C<reads()>, and a value every backend has already missed is
755: remembered in L<Geo::Coder::Free::MissCache>, one process-level cache that all
756: backends and instances share, whose entries expire after an hour and can be
757: emptied early with L</clear_miss_cache>. A later batch in the same process is
758: answered from those caches and so reports the reads it needed, often none, in
759: place of the ones the first batch made.
760:
761: =item * C<reads> - how many reads each backend made for the batch, keyed by the
762: name C<provenance()> reports that backend under: C<MaxMind>, C<Local> or
763: C<OpenAddresses>. Every backend the object holds is named, at zero for one that
764: read nothing, so a batch answered from a cache names its backends at zero rather
765: than naming none. C<lookups> counts values and this counts reads - a value that
766: made two backends read is one lookup and a read for each of them, so these
767: numbers can add up to more than C<lookups> - and the two vocabularies are the
768: same, so a caller can put the backend that read a value beside the backend a
769: result came from.
770:
771: =item * C<resolved> - how many distinct values produced a point.
772:
773: =item * C<unresolved> - how many produced nothing, the refused values included.
774:
775: =item * C<unresolved_values> - those values, in the order they were first seen,
776: so that a caller can record them as tried-and-failed rather than retrying them
777: on every request. Section 2.5 sketches this list under the name C<unresolved>;
778: here C<unresolved> is the count a "geocoded N of M" message needs and the values
779: themselves are C<unresolved_values>.
780:
781: =back
782:
783: C<resolved> and C<unresolved> always add up to the number of distinct values
784: given, and the five keys above are reserved: a value equal to one of them is
785: refused by name, before anything is looked up, rather than allowed to overwrite
786: a count.
787:
788: C<region> and C<on_error> are handed to every lookup and mean there what they
789: mean in C<geocode()>, including the rule that a backend exception is still an
790: exception - nothing here is wrapped in an C<eval>. C<on_error> defaults to
791: C<'croak'> for the same reason it does there: a batch should not quietly plot
792: fewer points than its column holds unless the caller asked it to, and a caller
793: working down a dirty column asks for C<'undef'>. The two settings do not differ
794: in which values are looked up, only in what happens to a value the module
795: refuses: C<'croak'> dies on it with the message C<geocode()> gives for it,
796: C<'undef'> counts it unresolved and carries on.
797:
798: A batch of no values is not an error: the counts come back zero.
799:
800: The values are treated as a set. Repeating a value in the input looks it up
801: once and every row is keyed to the same point object, so a column that repeats
802: the same handful of places thousands of times costs the handful of lookups.
803:
804: =head3 API SPECIFICATION
805:
806: =head4 input
807:
808: # Input schema (Params::Validate::Strict) - values required, options optional
809: values => { type => 'arrayref' } # location strings, one per row
810: region => { type => 'scalar', optional => 1 } # ISO 3166-1 alpha-2 country code hint
811: on_error => { type => 'scalar', optional => 1 } # 'croak' (the default) or 'undef'
812:
813: =head4 output
814:
815: # Output schema (Return::Set)
816: { type => 'hashref' }
817: # $result->{$value} => { isa => 'Geo::Location::Point', optional => 1 }
818: # $result->{lookups} => { type => 'integer' }
819: # $result->{reads} => { type => 'hashref' }
820: # $result->{resolved} => { type => 'integer' }
821: # $result->{unresolved} => { type => 'integer' }
822: # $result->{unresolved_values} => { type => 'arrayref' }
823:
824: =head3 MESSAGES
825:
826: Usage: ...::geocode_many(...) No values argument, one that is not an arrayref,
827: or options that are neither a list nor a hashref.
828: invalid location to geocode() A value refused under on_error => 'croak', with the
829: message geocode() gives for that value.
830: on_error must be ... on_error is neither 'croak' nor 'undef'.
831: <value> is one of the keys ... A value spelled like one of the five returned keys.
832:
833: =head3 EXAMPLE
834:
835: my $geo = Geo::Coder::Free->new(openaddr => $ENV{OPENADDR_HOME});
836:
837: # A log file's place column: repeats, a few unreadable values, one empty cell.
838: my @places = ('Ramsgate, Kent', '0044', 'Nowhere, Imaginary, ZZ',
839: 'Ramsgate, Kent', undef);
840:
841: my $batch = $geo->geocode_many(\@places, region => 'GB', on_error => 'undef');
842:
843: printf "geocoded %d of %d distinct places in %d lookups\n",
844: $batch->{'resolved'},
845: $batch->{'resolved'} + $batch->{'unresolved'},
846: $batch->{'lookups'}; # 4 distinct places, 2 of them looked up
847:
848: # Which backend did the reading, and which answered from a cache: a backend
849: # at zero answered nothing by reading.
850: while (my ($backend, $reads) = each %{$batch->{'reads'}}) {
851: print "$backend read $reads times\n";
852: }
853:
854: for my $place (@{$batch->{'unresolved_values'}}) {
855: warn "no coordinates for '" . (defined($place) ? $place : '') . "'\n";
856: }
857:
858: =cut
859:
860: sub geocode_many {
861: my $self = shift;
862:
863: # The same calling conventions as the rest of the module, with the batch's
864: # own spelling for the plain-function form: the values are always an
865: # arrayref, so that is what a function call arrives holding.
866: if (!Scalar::Util::blessed($self)) {
867: return __PACKAGE__->new()->geocode_many($self, @_)
868: if ref($self) eq 'ARRAY';
869: Carp::croak(_i18n('usage_geocode_many', __PACKAGE__)) unless @_;
870: return __PACKAGE__->new()->geocode_many(@_);
871: }
872:
873: my $values = shift;
874: Carp::croak(_i18n('usage_geocode_many', __PACKAGE__)) unless ref($values) eq 'ARRAY';
875:
876: # Options are a hashref or an even list. A lone scalar is neither: there is
877: # no bare-string form of a batch, so it is refused rather than guessed at.
878: my $opts;
879: if (@_ == 1 && ref($_[0]) eq 'HASH') {
880: $opts = { %{$_[0]} };
881: } elsif ((scalar @_) % 2 == 0 && !grep { ref } @_) {
882: $opts = { @_ };
883: } else {
884: Carp::croak(_i18n('usage_geocode_many', __PACKAGE__));
885: }
886:
887: my $on_error = $opts->{'on_error'} // 'croak';
888: Carp::croak(_i18n('invalid_on_error', __PACKAGE__, $on_error))
889: unless $on_error eq 'croak' || $on_error eq 'undef';
890:
891: my $region = $opts->{'region'};
892:
893: # A value spelled like one of the keys this method answers with would
894: # overwrite it, so it is refused by name - before anything is looked up,
895: # rather than after a count has been lost.
896: my %reserved = map { $_ => 1 } @RESULT_KEYS;
897: for my $value (@{$values}) {
898: my $key = defined($value) ? $value : '';
899: Carp::croak(_i18n('reserved_location', __PACKAGE__, $key)) if $reserved{$key};
900: }
901:
902: my %results = map { $_ => 0 } @RESULT_KEYS;
903: $results{'unresolved_values'} = [];
904: $results{'reads'} = {};
905:
906: # A batch is a set: a value repeated in the input is looked up once, and
907: # every row holding it is keyed to the same object.
908: my %seen;
909: for my $value (@{$values}) {
910: # undef has no key of its own, and the empty string it shares is not a
911: # location either, so it is unresolved without reaching a backend.
912: my $key = defined($value) ? $value : '';
913: next if $seen{$key}++;
914:
915: # What geocode() makes of this value on this setting is asked of
916: # geocode() rather than written out again here: the guard, its message and
917: # the choice of backend are all its own. The backends' read counters are
918: # taken around the call, so the value is counted as a lookup only if one of
919: # them actually read: a value answered from a cache, or from a hand-curated
920: # entry, is not counted as work the batch did not do. Taking them per
921: # backend rather than in total says which backend read, and which answered
922: # from somewhere else: a read is added to the name of the backend that made
923: # it, so a value two backends read through is one lookup and a read each.
924: my $reads_before = $self->_backend_reads();
925: my $point = $self->geocode(
926: location => $key,
927: defined($region) ? (region => $region) : (),
928: on_error => $on_error,
929: );
930: my $reads_after = $self->_backend_reads();
931:
932: my $read = 0;
933: foreach my $backend (keys %{$reads_after}) {
934: my $reads = $reads_after->{$backend} - ($reads_before->{$backend} // 0);
935: next unless $reads > 0;
936: $results{'reads'}{$backend} += $reads;
937: $read += $reads;
938: }
939:
940: # A value the module refuses is not a lookup: it never reached a backend,
941: # so it does not count against the work a caller is capping.
942: $results{'lookups'}++ if _valid_location($key) && $read;
943: $results{$key} = $point;
944:
945: if (defined($point)) {
946: $results{'resolved'}++;
947: } else {
948: $results{'unresolved'}++;
949: push @{$results{'unresolved_values'}}, $value;
950: }
951: }
952:
953: # A backend that read nothing is reported as well, at zero: what a caller is
954: # asking is which backend read and which answered from a cache, and a backend
955: # left out of the answer cannot be told from one that answered from one.
956: # Local is built the first time a lookup falls through to it, so it appears
957: # once it has been consulted, whether or not it found anything.
958: foreach my $backend (keys %{$self->_backend_reads()}) {
959: $results{'reads'}{$backend} //= 0;
960: }
961:
962: return \%results;
963: }
964:
965: # How much reading each backend has done so far, by the name it is reported
966: # under. Counters rather than flags: the change across a call says which
967: # backend read for that value, and a backend that was never called cannot leave
968: # a stale flag behind. A backend object may not exist yet - Local is built on
969: # first use - and a test may hand the facade a stand-in, so each slot is asked
970: # only if it holds something that can answer.
971: sub _backend_reads {
972: my $self = shift;
973:
974: my %reads;
975: foreach my $slot (sort keys %_BACKEND_NAMES) {
976: my $backend = $self->{$slot} or next;
977: next unless Scalar::Util::blessed($backend) && $backend->can('reads');
978: $reads{$_BACKEND_NAMES{$slot}} = $backend->reads() // 0;
979: }
980:
981: return \%reads;
982: }
983:
984: =head2 clear_miss_cache
985:
986: Geo::Coder::Free->clear_miss_cache();
987: $geo->clear_miss_cache();
988: Geo::Coder::Free->clear_miss_cache('OpenAddresses');
989:
990: Forgets the lookups that found nothing. Each backend remembers a miss in
991: L<Geo::Coder::Free::MissCache>, one process-level cache that every backend and
992: every instance shares, so an address that is not in the data is searched for
993: once per process rather than once per call. Entries expire after an hour; this
994: method empties them now, for a caller that has rebuilt a database or downloaded
995: a data set while the process is running.
996:
997: With a scope - C<OpenAddresses>, C<Local> or C<MaxMind> - only that backend's
998: entries go; C<OpenAddresses> covers every OpenAddresses directory this process
999: has opened. A scope that names no backend, such as
1000: C<clear_miss_cache('Nonsense')>, forgets nothing and is not an error. Returns
1001: true.
1002:
1003: =head3 API SPECIFICATION
1004:
1005: =head4 input
1006:
1007: scope => { type => 'string', optional => 1 } # OpenAddresses, Local or MaxMind
1008:
1009: =head4 output
1010:
1011: { type => 'boolean', value => 1 }
1012:
1013: =cut
1014:
1015: sub clear_miss_cache {
1016: # The invocant is $_[0] and is not needed; anything after it is the scope.
1017: Geo::Coder::Free::MissCache->clear(@_ > 1 ? $_[1] : undef);
1018:
1019: return 1;
1020: }
1021:
1022: =head2 provenance
1023:
1024: my ($dist, $backend, $confidence, $accuracy) = $geocoder->provenance($point);
1025:
1026: or, in scalar context, the same four as a hashref:
1027:
1028: my $provenance = $geocoder->provenance($point);
1029: print $provenance->{'backend'};
1030:
1031: Where a result came from and how good it is, so that a caller does not have to
1032: reach for the hash keys itself:
1033:
1034: =over 4
1035:
1036: =item * C<$dist> - the distribution that produced the result: the C<geocoder>
1037: key, C<Geo::Coder::Free> for a result from this distribution, or the wrapper
1038: that passed it on, such as C<Geo::Coder::List>.
1039:
1040: =item * C<$backend> - which backend answered, in a short vocabulary: C<Local>,
1041: C<MaxMind>, C<OpenAddresses> or C<GeoNames>. This is the C<database> key with
1042: the distribution's prefix removed, so that the two spellings the backends use -
1043: C<Geo::Coder::Free::Local> from C<Local>, and C<MaxMind> from C<MaxMind> - speak
1044: the same vocabulary here. B<The C<database> key itself is unchanged>, for the
1045: consumers that already match on it.
1046:
1047: =item * C<$confidence> - how good the match was, 0.0 to 1.0, where 1.0 is exact.
1048:
1049: =item * C<$accuracy> - how precise the coordinate is: C<address>, C<street>,
1050: C<postcode>, C<locality>, C<city>, C<region> or C<country>.
1051:
1052: =back
1053:
1054: Returns an empty list - and an empty hashref in scalar context - when handed
1055: something that is not a result. A key a result does not carry comes back
1056: C<undef> rather than being guessed at, because a plain hash from another
1057: geocoder has no provenance to report.
1058:
1059: =head3 API SPECIFICATION
1060:
1061: =head4 input
1062:
1063: point => { type => 'hashref', optional => 1 } # a result, usually a Geo::Location::Point
1064:
1065: =head4 output
1066:
1067: # list context
1068: { type => 'array', min => 0, max => 4 } # dist, backend, confidence, accuracy; or ()
1069:
1070: # scalar context
1071: {
1072: type => 'hashref',
1073: schema => {
1074: dist => { type => 'string', optional => 1 },
1075: backend => { type => 'string', optional => 1 },
1076: confidence => { type => 'number', optional => 1, min => 0, max => 1 },
1077: accuracy => { type => 'string', optional => 1 },
1078: },
1079: } # {} when handed something that is not a result
1080:
1081: =cut
1082:
1083: sub provenance {
1084: my ($self, $point) = @_;
1085:
1086: return wantarray ? () : {} unless defined($point) && ref($point)
1087: && ((Scalar::Util::reftype($point) || '') eq 'HASH');
1088:
1089: my $dist = $point->{'geocoder'};
1090: my $backend = $point->{'database'};
1091: $backend =~ s/^Geo::Coder::Free::// if defined($backend);
1092:
1093: my %provenance = (
1094: dist => $dist,
1095: backend => $backend,
1096: confidence => $point->{'confidence'},
1097: accuracy => $point->{'accuracy'},
1098: );
1099:
1100: return wantarray ? @provenance{qw(dist backend confidence accuracy)} : \%provenance;
1101: }
1102:
1103: =head2 reverse_geocode
1104:
1105: =head3 SYNOPSIS
1106:
1107: my $loc = $geo->reverse_geocode(latlng => '51.3341,-1.4159');
1108:
1109: =head3 DESCRIPTION
1110:
1111: Translates a latitude/longitude pair back to a place name.
1112: B<Partially implemented>: the MaxMind backend does not return meaningful results.
1113: OpenAddresses is attempted first when available.
1114:
1115: =head3 API SPECIFICATION
1116:
1117: =head4 input
1118:
1119: # Input schema (Params::Validate::Strict) - latlng required
1120: latlng => { type => 'scalar' } # "$lat,$long" comma-separated decimal degrees
1121: # NOTE: separate lat/lon/long keys are NOT supported at the Geo::Coder::Free
1122: # (facade) level. When no OpenAddresses backend is configured, passing
1123: # lat/lon/long instead of latlng will croak "not yet supported".
1124: # To use separate coordinates call Geo::Coder::Free::Local::reverse_geocode.
1125:
1126: =head4 output
1127:
1128: # Output schema (Return::Set)
1129: { type => 'object', isa => 'Geo::Location::Point', optional => 1 }
1130:
1131: =cut
1132:
1133: sub reverse_geocode {
1134: my $self = shift;
1135:
1136: if (!ref($self)) {
1137: if (scalar @_) {
1138: return __PACKAGE__->new()->reverse_geocode(@_);
1139: } elsif (!defined($self)) {
1140: Carp::croak(_i18n('usage_reverse', __PACKAGE__));
1141: } elsif ($self eq __PACKAGE__) {
1142: Carp::croak(_i18n('usage_reverse', $self));
1143: }
1144: return __PACKAGE__->new()->reverse_geocode($self);
1145: } elsif (ref($self) eq 'HASH') {
1146: return __PACKAGE__->new()->reverse_geocode($self);
1147: }
1148:
1149: my %params = _normalize_args(_i18n('usage_reverse', __PACKAGE__), 'latlng', @_);
1150:
1151: # M8 (tautology - same pattern fixed in geocode): both arms are identical;
1152: # context propagates implicitly through return.
1153: if ($self->{'openaddr'}) {
1154: return $self->{'openaddr'}->reverse_geocode(\%params);
1155: }
1156: if ($params{'latlng'}) {
1157: return $self->{'maxmind'}->reverse_geocode(\%params);
1158: }
1159:
1160: Carp::croak(_i18n('reverse_unsupported'));
1161: }
1162:
1163: =head2 ua
1164:
1165: Does nothing. Present for drop-in compatibility with other Geo::Coder::* modules.
1166:
1167: =head3 API SPECIFICATION
1168:
1169: =head4 input
1170:
1171: # Input schema (Params::Validate::Strict)
1172: # any arguments are accepted and ignored
1173:
1174: =head4 output
1175:
1176: # Output schema (Return::Set)
1177: # nothing: an empty list, or undef in scalar context
1178:
1179: =cut
1180:
1181: sub ua { }
1182:
1183: =head2 run
1184:
1185: Command-line entry point. Use as:
1186:
1187: perl lib/Geo/Coder/Free.pm 1600 Pennsylvania Avenue NW, Washington DC
1188:
1189: The arguments are joined with spaces into one location, which is geocoded with
1190: OpenAddresses as well when C<OPENADDR_HOME> is set. The results are printed to
1191: standard output with L<Data::Dumper>. It croaks C<geocoding failed> if nothing
1192: is found.
1193:
1194: =head3 API SPECIFICATION
1195:
1196: =head4 input
1197:
1198: # Input schema (Params::Validate::Strict)
1199: class => { type => 'string' } # Geo::Coder::Free
1200: location => { type => 'arrayref', of => 'string' } # the words of the location, joined with spaces
1201:
1202: =head4 output
1203:
1204: # Output schema (Return::Set)
1205: { type => 'boolean' } # the return value of print; croaks if nothing was found
1206:
1207: =cut
1208:
1209: __PACKAGE__->run(@ARGV) unless caller();
1210:
1211: sub run {
1212: require Data::Dumper;
1213:
1214: my $class = shift;
1215: my $location = join ' ', @_;
1216:
1217: my @rc = $ENV{'OPENADDR_HOME'}
1218: ? $class->new(openaddr => $ENV{'OPENADDR_HOME'})->geocode($location)
1219: : $class->new()->geocode($location);
1220:
1221: Carp::croak(_i18n('geocoding_failed', $0)) unless @rc;
1222:
1223: print Data::Dumper->new([\@rc])->Dump();
1224: }
1225:
1226: # -----------------------------------------------------------------------
1227: # Private helpers
1228: # -----------------------------------------------------------------------
1229:
1230: # The rule geocode() rejects on - whether a string could be a location at all -
1231: # is defined in Geo::Location::Predicate, and imported above. It is not
1232: # written out here as well: the backends ask the same question of what a caller
1233: # hands them, so one definition is what keeps valid_location(), the guard that
1234: # croaks and the backends from disagreeing about it.
1235:
1236: # Purpose: Return a formatted message string from the message table.
1237: # Falls back to the raw key so call sites never die on a missing key.
1238: # Entry: $key - message key; @args - sprintf format arguments.
1239: # Exit: Formatted string.
1240: sub _i18n {
1241: my ($key, @args) = @_;
1242: my $fmt = $_MESSAGES{$key} // $key;
1243: return @args ? sprintf($fmt, @args) : $fmt;
1244: }
1245:
1246: # Purpose: Normalise the four calling conventions accepted by geocode/reverse_geocode:
1247: # hashref, even-length key-val list, odd-length list, single bare string.
1248: # Entry: $error_msg - message to croak if a non-hash reference is passed.
1249: # $bare_key - hash key to assign to a single bare string argument
1250: # ('location' for geocode, 'latlng' for reverse_geocode).
1251: # @args - raw @_ after $self has been shifted.
1252: # Exit: Flat key-value %params hash.
1253: # Side Effects: None; croaks on unsupported reference argument.
1254: sub _normalize_args {
1255: my ($error_msg, $bare_key, @args) = @_;
1256: return %{$args[0]} if ref($args[0]) eq 'HASH';
1257: Carp::croak($error_msg) if ref($args[0]);
1258: return @args if @args && @args % 2 == 0;
1259: return ($bare_key => $args[0]) if @args == 1;
1260: # M7 (logic-gap closure): odd-count > 1 is not a valid calling convention.
1261: # The former code silently returned () here, discarding the trailing elements.
1262: # Fail fast instead of propagating a malformed argument list downstream.
1263: Carp::croak($error_msg) if @args;
1264: return ();
1265: }
1266:
1267: # Purpose: Try to geocode each candidate place string via the openaddr backend,
1268: # annotating every hit with location/text/confidence metadata and
1269: # memoising misses to avoid re-querying the same failing string.
1270: # Entry: $self - geocoder instance with {openaddr} and {scantext_misses}.
1271: # $candidates - arrayref of place-name strings to probe.
1272: # $region - optional ISO country code appended to each candidate.
1273: # $confidence - numeric confidence score assigned to all hits.
1274: # $scantext - original free text (stored inside each result object).
1275: # Exit: Arrayref of Geo::Location::Point objects; empty arrayref on no hits.
1276: # Side Effects: Populates $self->{scantext_misses} for every non-matching place.
1277: sub _resolve_scan_candidates {
1278: my ($self, $candidates, $region, $confidence, $scantext) = @_;
1279: my @results;
1280: for my $place (@{$candidates}) {
1281: my $location = $region ? "$place, $region" : $place;
1282: next if $self->{'scantext_misses'}{$location};
1283: my @res = grep { defined } $self->{'openaddr'}->geocode($location);
1284: if (@res) {
1285: for my $entry (@res) {
1286: $entry->{'location'} = $location;
1287: $entry->{'text'} = $scantext;
1288: $entry->{'confidence'} = $confidence;
1289: }
1290: push @results, @res;
1291: } else {
1292: $self->{'scantext_misses'}{$location} = 1;
1293: }
1294: }
1295: return \@results;
1296: }
1297:
1298: # Purpose: Slide a window of $n words across $text and return all comma-joined
1299: # n-grams, excluding pure-numeric tokens and stopwords. A token of
1300: # digits only is not a word of a place name, which is the question
1301: # Geo::Location::Predicate::_valid_location() answers, so it is asked
1302: # there rather than spelled out here (t/location-rule.t fails if the
1303: # rule is written out again).
1304: # Replaces the former _find_word_triplets ($n=3) and
1305: # _find_word_duplets ($n=2) which were identical except for window size
1306: # and the duplets version was missing the lc() normalisation on stopwords.
1307: # Entry: $text - raw string; $n - window size; $stop - stopword hashref.
1308: # Exit: Flat list of n-gram strings.
1309: sub _find_word_ngrams {
1310: my ($text, $n, $stop) = @_;
1311: $text =~ s/,+/ /g;
1312: $text =~ s/\s+/ /g;
1313: $text =~ s/^\s+|\s+$//g;
1314: my @words = grep { !$stop->{lc $_} && _valid_location($_) } split /\s+/, $text;
1315: my @ngrams;
1316: for my $i (0 .. $#words - ($n - 1)) {
1317: push @ngrams, join ', ', @words[$i .. $i + $n - 1];
1318: }
1319: return @ngrams;
1320: }
1321:
1322: # Purpose: Extract structured US street addresses from free text.
1323: # Entry: $text - arbitrary string.
1324: # Exit: List of full address strings matching the US pattern.
1325: sub _find_us_addresses {
1326: my $text = shift;
1327: my @addresses;
1328: my $re = qr/
1329: \b \d{1,5} \s+ # house number
1330: (?:[A-Za-z0-9]+(?:\s+[A-Za-z0-9]+){0,6}) \s+ # street name: 1-7 words (bounded)
1331: (?:Avenue|Ave\.?|Boulevard|Blvd\.?|Road|Rd\.?|Lane|Ln\.?|Drive|Dr\.?|Street|St\.?)
1332: (?:\s+[A-Za-z]{2})? ,\s*
1333: (?:[A-Za-z]+(?:\s+[A-Za-z]+){0,3}) ,\s* # city: 1-4 words
1334: [A-Z]{2} \s* (?:\d{5}(?:-\d{4})?)? \b # state + optional zip
1335: /x;
1336: while ($text =~ /$re/g) {
1337: push @addresses, $&;
1338: }
1339: return @addresses;
1340: }
1341:
1342: # Purpose: Extract British-style addresses from free text.
1343: # Entry: $text - arbitrary string.
1344: # Exit: List of trimmed address strings.
1345: sub _find_gb_addresses {
1346: my $text = shift;
1347: my @addresses;
1348: # ReDoS fix: the former pattern used \s*,?\s* between all parts and
1349: # [A-Za-z\s'-]+ groups - with commas optional, the engine must try all
1350: # possible splits of a long string among 5 overlapping space-containing
1351: # groups, causing exponential backtracking. Making commas mandatory
1352: # eliminates the ambiguity entirely: UK postal addresses always have commas.
1353: # All groups are non-capturing (only $& is used).
1354: my $re = qr/
1355: \b
1356: (?:\d{1,5} | [\w'-]+) # house number or single-word name
1357: \s+
1358: (?:[\w'-]+(?:\s+[\w'-]+){0,5}) # street name (up to 6 words)
1359: \s*,\s* # COMMA REQUIRED - kills the ambiguity
1360: (?:[\w'-]+(?:\s+[\w'-]+){0,3}) # town (up to 4 words)
1361: \s*,\s* # COMMA REQUIRED
1362: (?:[\w'-]+(?:\s+[\w'-]+){0,3}) # county (up to 4 words)
1363: \s*,\s* # COMMA REQUIRED
1364: (?:[\w'-]+(?:\s+[\w'-]+){0,2}) # country (up to 3 words)
1365: \b
1366: /x;
1367: while ($text =~ /$re/g) {
1368: (my $addr = $&) =~ s/[,\s]+$//;
1369: push @addresses, $addr;
1370: }
1371: return @addresses;
1372: }
1373:
1374: # Purpose: Extract Canadian street addresses from free text.
1375: # Entry: $text - arbitrary string.
1376: # Exit: List of full address strings matching the Canadian pattern.
1377: sub _find_ca_addresses {
1378: my $text = shift;
1379: my @addresses;
1380: my $re = qr/
1381: \b \d{1,5} \s+ # house number
1382: (?:[A-Za-z0-9]+(?:\s+[A-Za-z0-9]+){0,6}) \s+ # street name: 1-7 words (bounded)
1383: (?:Avenue|Ave\.?|Boulevard|Blvd\.?|Road|Rd\.?|Lane|Ln\.?|Drive|Dr\.?|Street|St\.?|Circle|Crescent|Cres\.?)
1384: \s*,\s*
1385: (?:[A-Za-z]+(?:\s+[A-Za-z]+){0,3}) \s*,\s* # city: 1-4 words
1386: [A-Z]{2} \s*,?\s*
1387: (?:[A-Z]\d[A-Z]\s?\d[A-Z]\d)? \b # optional postal code
1388: /x;
1389: while ($text =~ /$re/g) {
1390: push @addresses, $&;
1391: }
1392: return @addresses;
1393: }
1394:
1395: # _normalize and _abbreviate are imported from Geo::Coder::Free::Utils.
1396:
1397: =head1 GETTING STARTED
1398:
1399: To download, import and set up the local database:
1400: before running C<make>, but after running C<perl Makefile.PL>, follow these instructions.
1401:
1402: Optionally set C<OPENADDR_HOME> to point to an empty directory and download the data from
1403: L<http://results.openaddresses.io> into that directory; and
1404: optionally set C<WHOSONFIRST_HOME> to point to an empty directory and download the data using
1405: L<https://github.com/nigelhorne/NJH-Snippets/blob/master/bin/wof-clone>.
1406: The script C<bin/download_databases> (see below) will do those for you.
1407: You do not need to download the MaxMind data - that is downloaded automatically.
1408:
1409: You will need to create the database used by C<Geo::Coder::Free>.
1410:
1411: Install L<App::csv2sqlite> and L<https://github.com/nigelhorne/NJH-Snippets>.
1412: Run C<bin/create_sqlite> - this converts the MaxMind "cities" database from CSV to SQLite.
1413:
1414: To use with MariaDB, set C<MARIADB_SERVER="$hostname;$port"> and
1415: C<MARIADB_USER="$user;$password"> (TODO: username/password should be asked for interactively).
1416: The code will use a database called C<geo_code_free>, which will be dropped and recreated if it exists.
1417: C<$user> needs only DROP, CREATE, SELECT, INSERT, and INDEX privileges on that database.
1418:
1419: The following optional steps download and install large databases.
1420: This will take a long time and use a lot of disc space.
1421:
1422: =over 4
1423:
1424: =item 1
1425:
1426: C<mkdir $WHOSONFIRST_HOME; cd $WHOSONFIRST_HOME> then run C<wof-clone> from NJH-Snippets.
1427:
1428: This can take a long time because it contains many nested directories, which filesystem drivers
1429: can be slow to navigate (particularly on EXT4 and ZFS).
1430:
1431: =item 2
1432:
1433: Install L<https://github.com/dr5hn/countries-states-cities-database.git> into C<$DR5HN_HOME>.
1434: This data covers cities only, so it is not used when C<OSM_HOME> is set (OSM is far more
1435: comprehensive). Only Australia, Canada, and the US are imported, as the UK data is difficult
1436: to parse.
1437:
1438: =item 3
1439:
1440: Run C<bin/download_databases> - this downloads the Who's On First, OpenAddr, OpenStreetMap,
1441: and dr5hn databases.
1442: OpenStreetMap now uses PBF files, so you will need C<apt install osmium-tool> first.
1443: Check the values of C<OSM_HOME>, C<OPENADDR_HOME>, C<DR5HN_HOME> and C<WHOSONFIRST_HOME>
1444: within that script and adjust them for your setup.
1445: The C<Makefile.PL> file downloads the MaxMind database automatically, as it is not optional.
1446:
1447: =item 4
1448:
1449: Run C<bin/create_db> - this creates the database used by C<Geo::Coder::Free> from the data you
1450: have just downloaded.
1451: The database is called C<openaddr.sql> for historical reasons (before Who's On First was added);
1452: it actually contains data from all sources above.
1453:
1454: =back
1455:
1456: Now you are ready to run C<make>.
1457: See the comment at the start of C<createdatabase.PL> for further details.
1458:
1459: =head1 MORE INFORMATION
1460:
1461: I have written several Perl genealogy programs including
1462: L<gedcom|https://github.com/nigelhorne/gedcom> and
1463: L<ged2site|https://github.com/nigelhorne/ged2site>.
1464: One of the things these do is check the validity of a family tree, including verifying place-names.
1465: Of course places do change names and spelling becomes more consistent over the years, but the vast
1466: majority remain the same - enough to make computerised verification worthwhile.
1467:
1468: =head1 BUGS
1469:
1470: Some lookups fail. Please file a bug report at
1471: L<https://rt.cpan.org/NoAuth/Bugs.html?Dist=Geo-Coder-Free>.
1472:
1473: The MaxMind data contains cities only.
1474: The OpenAddresses data does not cover the whole globe.
1475: C<London, England> cannot be parsed yet.
1476:
1477: =head1 SEE ALSO
1478:
1479: =over 4
1480:
1481: =item * L<Configure an Object at Runtime|Object::Configure>
1482:
1483: =item * L<Test Dashboard|https://nigelhorne.github.io/Geo-Coder-Free/coverage/>
1484:
1485: =back
1486:
1487: L<Geo::Coder::Free::Local>, L<Geo::Coder::Free::MaxMind>,
1488: L<Geo::Coder::Free::OpenAddresses>,
1489: L<https://openaddresses.io/>, L<https://www.maxmind.com/>,
1490: L<https://www.geonames.org/>, L<https://www.whosonfirst.org/>.
1491:
1492: =head1 AUTHOR
1493:
1494: Nigel Horne C<< <njh@nigelhorne.com> >>
1495:
1496: =head1 FORMAL SPECIFICATION
1497:
1498: =head2 new
1499:
1500: GeoCoderFreeState ::= ⟨⟨ maxmind : MaxMind_Geocoder;
1501: openaddr : OpenAddr_Geocoder | undef;
1502: alternatives: Map[STRING → STRING];
1503: cache : Cache | undef ⟩⟩
1504:
1505: OpenAddrState ::= ⟨⟨ openaddr : PATH;
1506: cache : Cache | undef;
1507: libpostal : BOOLEAN ⟩⟩
1508:
1509: Init : Params → GeoCoderFreeState
1510: ∀ p : Params •
1511: let oa_path == p.openaddr ∨ env.OPENADDR_HOME •
1512: GeoCoderFreeState.openaddr = if oa_path ≠∅
1513: then OpenAddresses(oa_path, p.cache, p.libpostal)
1514: else undef fi
1515: ∧ (oa_path ≠∅ ⟹ GeoCoderFreeState.openaddr.libpostal =
1516: if p.libpostal = undef then true else p.libpostal fi)
1517:
1518: -- libpostal = false: Geo::libpostal is never loaded or called
1519: ∀ s : OpenAddrState • ¬ s.libpostal ⟹ Geo::libpostal ∉ loaded_by(s.geocode)
1520:
1521: =head2 valid_location
1522:
1523: ValidLocation : Any → BOOLEAN
1524: ∀ x : Any •
1525: ValidLocation(x) = (x ≠∅ ∧ ¬ ref(x) ∧ x ≠"" ∧ x ∋ non-digit)
1526:
1527: =head2 geocode
1528:
1529: ErrorMode ::= "croak" | "undef"
1530:
1531: Geocode : Address × Region? × ErrorMode? → Point?
1532: ∀ addr : Address; r : Region?; e : ErrorMode? •
1533: let mode == e ∨ "croak" •
1534: let backends == (openaddr ≠undef ⟹ [OpenAddresses, Local, MaxMind])
1535: ∧ (openaddr = undef ⟹ [MaxMind]) •
1536: ¬ ValidLocation(addr) ⟹ (mode = "undef" ⟹ result = ∅ | mode = "croak" ⟹ ⊥)
1537: ∧ ValidLocation(addr) ⟹
1538: result = first { defined } map { b.geocode(addr, r) } backends
1539:
1540: =head2 provenance
1541:
1542: Provenance : Point? → (DISTRIBUTION?, BACKEND?, CONFIDENCE?, ACCURACY?)
1543: ∀ p : Point? •
1544: Provenance(p) = if p ≠∅ ∧ is_a_result(p)
1545: then ⟨⟨ p.geocoder,
1546: strip("Geo::Coder::Free::", p.database),
1547: p.confidence,
1548: p.accuracy ⟩⟩
1549: else ∅ fi
1550:
1551: =head1 LICENSE AND COPYRIGHT
1552:
1553: Copyright 2017-2026 Nigel Horne. Licensed under GPL2 for personal use.
1554:
1555: This product uses GeoLite2 data created by MaxMind,
1556: available from L<https://www.maxmind.com/>.
1557:
1558: =cut
1559:
1560: 1;
1561:
1562: # Common mappings for looser lookups, one "from = to" per line; lines starting
1563: # with # are ignored. They are tried only after every backend has missed, so
1564: # "Tyne and Wear" is a last try under the county GeoNames uses, not a claim
1565: # that all of Tyne and Wear is in North Tyneside. A future version should load
1566: # these from an external, user-editable config file. See also Local.pm's
1567: # %alternatives.
1568: __DATA__
1569: St Lawrence, Thanet, Kent = Ramsgate, Kent
1570: St Peters, Thanet, Kent = Broadstairs, Kent
1571: Minster, Thanet, Kent = Ramsgate, Kent
1572: Tyne and Wear = Borough of North Tyneside