Form 477 update for 2022+ reporting (2020 census data), RT#86245 (New FS::Misc::Geo::get_censustract_uscensus subroutine contributed by Jim Lucas <jlucas@cmsws.com>, thanks!)

6 files changed, 108 insertions, 17 deletions

diff --git a/FS/FS/Conf.pm b/FS/FS/Conf.pm
index fc9708b..fd6f668 100644
--- a/FS/FS/Conf.pm
+++ b/FS/FS/Conf.pm
@@ -4624,13 +4624,25 @@ and customer address. Include units.',
 
   {
     'key'         => 'census_year',
-    'section'     => 'addresses',
-    'description' => 'The year to use in census tract lookups.  NOTE: you need to select 2012 or 2013 for Year 2010 Census tract codes.  A selection of 2011 provides Year 2000 Census tract codes.  Use the freeside-censustract-update tool if exisitng customers need to be changed.',
+    'section'     => 'deprecated',
+    'description' => 'Deprecated.  Used to control the year used for census lookups.  2020 census data is now the default.  Use the freeside-censustract-update tool if exisitng customers need to be changed.  See the <a href ="#census_legacy">census_legacy</a> configuration option if you need old census data to re-file pre-2022 FCC 477 reports.',
     'type'        => 'select',
     'select_enum' => [ qw( 2017 2016 2015 ) ],
   },
 
   {
+    'key'         => 'census_legacy',
+    'section'     => 'addresses',
+    'description' => 'Use old census data (and source).  Should only be needed if re-filing pre-2022 FCC 477 reports.',
+    'type'        => 'select',
+    'select_hash' => [ '' => 'Disabled (2020)',
+                       '2015' => '2015',
+                       '2016' => '2016',
+                       '2017' => '2017',
+                     ],
+  },
+
+  {
     'key'         => 'tax_district_method',
     'section'     => 'taxation',
     'description' => 'The method to use to look up tax district codes.',
diff --git a/FS/FS/GeocodeCache.pm b/FS/FS/GeocodeCache.pm
index 7829c4d..430a90f 100644
--- a/FS/FS/GeocodeCache.pm
+++ b/FS/FS/GeocodeCache.pm
@@ -117,16 +117,15 @@ the only one is 'ffiec'.
 sub set_censustract {
   my $self = shift;
 
-  if ( $self->get('censustract') =~ /^\d{9}\.\d{2}$/ ) {
+  if ( $self->get('censustract') =~ /^\d{9}(\.\d{2}|\d{6})$/ ) {
     return $self->get('censustract');
   }
-  my $censusyear = $conf->config('census_year');
-  return if !$censusyear;
 
-  my $method = 'ffiec';
-  # configurable censustract-only lookup goes here if it's ever needed.
+  my $year = $conf->config('census_legacy') || 2020;
+  my $method = ($year==2020) ? 'uscensus' : 'ffiec';
+
   $method = "get_censustract_$method";
-  my $censustract = eval { FS::Misc::Geo->$method($self, $censusyear) };
+  my $censustract = eval { FS::Misc::Geo->$method($self, $year) };
   $self->set("censustract_error", $@);
   $self->set("censustract", $censustract);
 }
diff --git a/FS/FS/Misc/Geo.pm b/FS/FS/Misc/Geo.pm
index bc020a2..599b2a0 100644
--- a/FS/FS/Misc/Geo.pm
+++ b/FS/FS/Misc/Geo.pm
@@ -38,6 +38,10 @@ Given a location hash (see L<FS::location_Mixin>) and a census map year,
 returns a census tract code (consisting of state, county, and tract 
 codes) or an error message.
 
+Data source: Federal Financial Institutions Examination Council
+
+Note: This is the old method for pre-2022 (census year 2020) reporting.
+
 =cut
 
 sub get_censustract_ffiec {
@@ -105,6 +109,79 @@ sub get_censustract_ffiec {
   }
 }
 
+=item get_censustract_uscensus LOCATION YEAR
+
+Given a location hash (see L<FS::location_Mixin>) and a census map year,
+returns a census tract code (consisting of state, county, tract, and block
+codes) or an error message.
+
+Data source: US Census Bureau
+
+This is the new method for 2022+ (census year 2020) reporting.
+
+=cut
+
+sub get_censustract_uscensus {
+  my $class    = shift;
+  my $location = shift;
+  my $year     = shift || 2020;
+
+  if ( length($location->{country}) and uc($location->{country}) ne 'US' ) {
+    return '';
+  }
+
+  warn Dumper($location, $year) if $DEBUG;
+
+  my $url = 'https://geocoding.geo.census.gov/geocoder/geographies/address?';
+
+  my $query_hash = {
+                     street     => $location->{address1},
+                     city       => $location->{city},
+                     state      => $location->{state},
+                     benchmark  => 'Public_AR_Current',
+                     vintage    => 'Census'.$year.'_Current',
+                     format     => 'json',
+                   };
+
+  my $full_url = URI->new($url);
+  $full_url->query_form($query_hash);
+
+  warn "Full Request URL: \n".$full_url if $DEBUG;
+
+  my $ua = new LWP::UserAgent;
+  my $res = $ua->get( $full_url );
+
+  warn $res->as_string if $DEBUG > 2;
+
+  if (!$res->is_success) {
+    die 'Census tract lookup error: '.$res->message;
+  }
+
+  local $@;
+  my $content = eval { decode_json($res->content) };
+  die "Census tract JSON error: $@\n" if $@;
+
+  warn Dumper($content) if $DEBUG;
+
+  if ( $content->{result}->{addressMatches} ) {
+
+    my $tract = $content->{result}->{addressMatches}[0]->{geographies}->{'Census Blocks'}[0]->{GEOID};
+    return $tract;
+
+  } else {
+
+    my $error = 'Lookup failed, but with no status message.';
+
+    if ( $content->{errors} ) {
+      $error = join("\n", $content->{errors});
+    }
+
+    die "$error\n";
+
+  }
+}
+
+
 #sub get_district_methods {
 #  ''         => '',
 #  'wa_sales' => 'Washington sales tax',
@@ -660,6 +737,7 @@ sub subloc_address2 {
   ($subloc, $addr2);
 }
 
+#is anyone still using this?
 sub standardize_melissa {
   my $class = shift;
   my $location = shift;
diff --git a/FS/FS/cust_location.pm b/FS/FS/cust_location.pm
index a2bda71..28278cb 100644
--- a/FS/FS/cust_location.pm
+++ b/FS/FS/cust_location.pm
@@ -252,7 +252,7 @@ sub insert {
   }
 
   if ( $self->censustract ) {
-    $self->set('censusyear' => $conf->config('census_year') || 2012);
+    $self->set('censusyear' => $conf->config('census_legacy') || 2020);
   }
 
   my $oldAutoCommit = $FS::UID::AutoCommit;
@@ -419,10 +419,13 @@ sub check {
   ;
   return $error if $error;
   if ( $self->censustract ne '' ) {
-    $self->censustract =~ /^\s*(\d{9})\.?(\d{2})\s*$/
-      or return "Illegal census tract: ". $self->censustract;
-
-    $self->censustract("$1.$2");
+    if ( $self->censustract =~ /^\s*(\d{9})\.?(\d{2})\s*$/ ) { #old
+      $self->censustract("$1.$2");
+    } elsif ($self->censustract =~ /^\s*(\d{15})\s*$/ ) { #new
+      $self->censustract($1);
+    } else {
+      return "Illegal census tract: ". $self->censustract;
+    }
   }
 
   #yikes... this is ancient, pre-dates cust_location and will be harder to
@@ -832,7 +835,7 @@ sub process_censustract_update {
     qsearchs( 'cust_location', { locationnum => $locationnum })
       or die "locationnum '$locationnum' not found!\n";
 
-  my $new_year = $conf->config('census_year') or return;
+  my $new_year = $conf->config('census_legacy') || 2020;
   my $loc = FS::GeocodeCache->new( $cust_location->location_hash );
   $loc->set_censustract;
   my $error = $loc->get('censustract_error');
diff --git a/FS/bin/freeside-censustract-update b/FS/bin/freeside-censustract-update
index f9b6d11..27a17be 100755
--- a/FS/bin/freeside-censustract-update
+++ b/FS/bin/freeside-censustract-update
@@ -18,8 +18,7 @@ $FS::UID::AutoCommit = 0;
 my $dbh = dbh;
 
 my $conf = FS::Conf->new;
-my $current_year = $conf->config('census_year') 
-  or die "No current census year configured.\n";
+my $current_year = $conf->config('census_legacy') || '2020';
 my $date = str2time($opt{d}) if $opt{d};
 $date ||= time;
 # This now operates on cust_location, not cust_main.
diff --git a/httemplate/misc/confirm-censustract.html b/httemplate/misc/confirm-censustract.html
index 0f115e5..8535c14 100644
--- a/httemplate/misc/confirm-censustract.html
+++ b/httemplate/misc/confirm-censustract.html
@@ -103,7 +103,7 @@ my %location = (
 my $old_tract = $q->{$pre.'censustract'};
 my $cache = eval { FS::GeocodeCache->new(%location) };
 $cache->set_censustract;
-my $year = FS::Conf->new->config('census_year');
+my $year = FS::Conf->new->config('census_legacy') || '2020';
 my $new_tract = $cache->get('censustract');
 my $error = $cache->get('censustract_error');