Merge branch 'master' of git.freeside.biz:/home/git/freeside

5 files changed, 83 insertions, 116 deletions

diff --git a/FS/FS/Misc/Geo.pm b/FS/FS/Misc/Geo.pm
index a387aca9d..dbc383a14 100644
--- a/FS/FS/Misc/Geo.pm
+++ b/FS/FS/Misc/Geo.pm
@@ -6,8 +6,7 @@ use vars qw( $DEBUG @EXPORT_OK $conf );
 use LWP::UserAgent;
 use HTTP::Request;
 use HTTP::Request::Common qw( GET POST );
-use HTTP::Cookies;
-use HTML::TokeParser;
+use JSON;
 use URI::Escape 3.31;
 use Data::Dumper;
 use FS::Conf;
@@ -29,7 +28,7 @@ FS::Misc::Geo - routines to fetch geographic information
 
 =over 4
 
-=item get_censustract LOCATION YEAR
+=item get_censustract_ffiec LOCATION YEAR
 
 Given a location hash (see L<FS::location_Mixin>) and a census map year,
 returns a census tract code (consisting of state, county, and tract 
@@ -41,6 +40,7 @@ sub get_censustract_ffiec {
   my $class = shift;
   my $location = shift;
   my $year  = shift;
+  $year ||= 2013;
 
   if ( length($location->{country}) and uc($location->{country}) ne 'US' ) {
     return '';
@@ -48,102 +48,57 @@ sub get_censustract_ffiec {
 
   warn Dumper($location, $year) if $DEBUG;
 
-  my $url = 'http://www.ffiec.gov/Geocode/default.aspx';
-
-  my $return = {};
-  my $error = '';
+  # the old FFIEC geocoding service was shut down December 1, 2014.
+  # welcome to the future.
+  my $url = 'https://geomap.ffiec.gov/FFIECGeocMap/GeocodeMap1.aspx/GetGeocodeData';
+  # build the single-line query
+  my $single_line = join(', ', $location->{address1},
+                               $location->{city},
+                               $location->{state}
+                        );
+  my $hashref = { sSingleLine => $single_line, iCensusYear => $year };
+  my $request = POST( $url,
+    'Content-Type' => 'application/json; charset=utf-8',
+    'Accept' => 'application/json',
+    'Content' => encode_json($hashref)
+  );
 
-  my $ua = new LWP::UserAgent('cookie_jar' => HTTP::Cookies->new);
-  my $res = $ua->request( GET( $url ) );
+  my $ua = new LWP::UserAgent;
+  my $res = $ua->request( $request );
 
   warn $res->as_string
     if $DEBUG > 2;
 
   if (!$res->is_success) {
 
-    $error = $res->message;
-
-  } else {
-
-    my $content = $res->content;
-
-    my $p = new HTML::TokeParser \$content;
-    my $viewstate;
-    my $eventvalidation;
-    while (my $token = $p->get_tag('input') ) {
-      if ($token->[1]->{name} eq '__VIEWSTATE') {
-        $viewstate = $token->[1]->{value};
-      }
-      if ($token->[1]->{name} eq '__EVENTVALIDATION') {
-        $eventvalidation = $token->[1]->{value};
-      }
-      last if $viewstate && $eventvalidation;
-    }
-
-    if (!$viewstate or !$eventvalidation ) {
-
-      $error = "either no __VIEWSTATE or __EVENTVALIDATION found";
+    die "Census tract lookup error: ".$res->message;
 
-    } else {
-
-      my($zip5, $zip4) = split('-',$location->{zip});
-
-      $year ||= '2013';
-      my @ffiec_args = (
-        __VIEWSTATE => $viewstate,
-        __EVENTVALIDATION => $eventvalidation,
-        __VIEWSTATEENCRYPTED => '',
-        ddlbYear    => $year,
-        txtAddress  => $location->{address1},
-        txtCity     => $location->{city},  
-        ddlbState   => $location->{state},
-        txtZipCode  => $zip5,
-        btnSearch   => 'Search',
-      );
-      warn join("\n", @ffiec_args )
-        if $DEBUG > 1;
-
-      push @{ $ua->requests_redirectable }, 'POST';
-      $res = $ua->request( POST( $url, \@ffiec_args ) );
-      warn $res->as_string
-        if $DEBUG > 2;
-
-      unless ($res->code  eq '200') {
-
-        $error = $res->message;
-
-      } else {
-
-        my @id = qw( MSACode StateCode CountyCode TractCode );
-        $content = $res->content;
-        warn $res->content if $DEBUG > 2;
-        $p = new HTML::TokeParser \$content;
-        my $prefix = 'UcGeoResult11_lb';
-        my $compare =
-          sub { my $t=shift; scalar( grep { lc($t) eq lc("$prefix$_")} @id ) };
-
-        while (my $token = $p->get_tag('span') ) {
-          next unless ( $token->[1]->{id} && &$compare( $token->[1]->{id} ) );
-          $token->[1]->{id} =~ /^$prefix(\w+)$/;
-          $return->{lc($1)} = $p->get_trimmed_text("/span");
-        }
-
-        unless ( $return->{tractcode} ) {
-          warn "$error: $content ". Dumper($return) if $DEBUG;
-          $error = "No census tract found";
-        }
-        $return->{tractcode} .= ' '
-          unless $error || $JSON::VERSION >= 2; #broken JSON 1 workaround
+  }
 
-      } #unless ($res->code  eq '200')
+  local $@;
+  my $content = eval { decode_json($res->content) };
+  die "Census tract JSON error: $@\n" if $@;
 
-    } #unless ($viewstate)
+  if ( !exists $content->{d}->{sStatus} ) {
+    die "Census tract response is missing a status indicator.\nThis is an FFIEC problem.\n";
+  }
+  if ( $content->{d}->{sStatus} eq 'Y' ) {
+    # success
+    # this also contains the (partial) standardized address, correct zip 
+    # code, coordinates, etc., and we could get all of them, but right now
+    # we only want the census tract
+    my $tract = join('', $content->{d}->{sStateCode},
+                         $content->{d}->{sCountyCode},
+                         $content->{d}->{sTractCode});
+    return $tract;
 
-  } #unless ($res->code  eq '200')
+  } else {
 
-  die "FFIEC Geocoding error: $error\n" if $error;
+    my $error = $content->{d}->{sMsg}
+            ||  'FFIEC lookup failed, but with no status message.';
+    die "$error\n";
 
-  $return->{'statecode'} .  $return->{'countycode'} .  $return->{'tractcode'};
+  }
 }
 
 #sub get_district_methods {
diff --git a/httemplate/edit/cust_main/bottomfixup.js b/httemplate/edit/cust_main/bottomfixup.js
index 8aef1e72c..6a9deb92a 100644
--- a/httemplate/edit/cust_main/bottomfixup.js
+++ b/httemplate/edit/cust_main/bottomfixup.js
@@ -100,14 +100,13 @@ function copyelement(from, to) {
   //alert(from + " (" + from.type + "): " + to.name + " => " + to.value);
 }
 
-% # the value in pre+'censustract' is the confirmed censustract; if it's set,
-% # and the user hasn't changed it manually, skip this
+% # the value in pre+'censustract' is the confirmed censustract (either from
+% # the previous saved record, or from address standardization (if the backend
+% # supports it), or from an aborted previous submit. only need to reconfirm
+% # if it's empty.
 function confirm_censustract(pre) {
   var cf = document.CustomerForm;
-  if ( cf.elements[pre+'censustract'].value == '' ||
-         cf.elements[pre+'enter_censustract'].value != 
-         cf.elements[pre+'censustract'].value )
-  {
+  if ( cf.elements[pre+'censustract'].value == '' ) {
     var address_info = form_address_info();
     address_info[pre+'latitude']  = cf.elements[pre+'latitude'].value;
     address_info[pre+'longitude'] = cf.elements[pre+'longitude'].value;
diff --git a/httemplate/elements/location.html b/httemplate/elements/location.html
index 5cdc424a7..214a7d5f2 100644
--- a/httemplate/elements/location.html
+++ b/httemplate/elements/location.html
@@ -59,12 +59,7 @@ Example:
     </TR>
 
 % } else {
-
-    <INPUT TYPE     = "hidden"
-           NAME     = "<%$pre%>locationname"
-           ID       = "<%$pre%>locationname"
-           VALUE    = "<% $object->get('locationname') |h %>"
-    >
+    <& hidden.html, field => $pre.'locationname', value => $object->get('locationname') &>
 
 % }
 
@@ -102,10 +97,7 @@ Example:
 
 % } else { # alternate format
 
-      <INPUT TYPE  = "hidden"
-             NAME  = "<%$pre%>address2"
-             VALUE = "<% $object->get('address2') |h %>"
-      >
+<& hidden.html, field => $pre.'address2', value => $object->get('address2') &>
 
 <TR>
     <<%$th%> ALIGN="right">Unit&nbsp;type&nbsp;and&nbsp;#</<%$th%>>
@@ -227,14 +219,14 @@ Example:
 </TR>
 % } else {
 %   foreach (qw(latitude longitude)) {
-<INPUT TYPE="hidden" NAME="<% $_ %>" ID="<% $_ %>" VALUE="<% $object->get($_) |h%>">
+<& hidden.html, field => $pre.$_, value => $object->get($_) &>
 %   }
 % }
-<INPUT TYPE="hidden" NAME="<%$pre%>coord_auto" VALUE="<% $object->coord_auto %>">
-
-<INPUT TYPE="hidden" NAME="<%$pre%>geocode" VALUE="<% $object->geocode %>">
-<INPUT TYPE="hidden" NAME="<%$pre%>censustract" VALUE="<% $object->censustract %>">
-<INPUT TYPE="hidden" NAME="<%$pre%>censusyear" VALUE="<% $object->censusyear %>">
+%
+% foreach (qw(coord_auto geocode censustract censusyear)) {
+  <& hidden.html, field => $pre.$_, value => $object->get($_) &>
+% }
+%
 % if ( $opt{enable_censustract} ) {
 <TR>
   <TD ALIGN="right">Census&nbsp;tract</TD>
@@ -259,7 +251,7 @@ Example:
     </TD>
   </TR>
 % } else {
-    <INPUT TYPE="hidden" ID="<%$pre%>" NAME="<%$pre%>district" VALUE="<% $object->district %>">
+  <& hidden.html, field => $pre.'district', value => $object->get('district') &>
 % }
 
 %# For address standardization:
@@ -267,11 +259,11 @@ Example:
 %# to re-standardize
 % foreach (qw(address1 city state country zip latitude
 %             longitude censustract district addr_clean) ) {
-<INPUT TYPE="hidden" NAME="old_<%$pre.$_%>" ID="old_<%$pre.$_%>" VALUE="<% $object->get($_) |h%>">
+<& hidden.html, field => 'old_'.$pre.$_, value => $object->get($_) &>
 % }
 %# Placeholders
-<INPUT TYPE="hidden" NAME="<%$pre%>cachenum" VALUE="">
-<INPUT TYPE="hidden" NAME="<%$pre%>addr_clean" VALUE="">
+<& hidden.html, field => $pre.'cachenum', value => '' &>
+<& hidden.html, field => $pre.'addr_clean', value => '' &>
 
 <SCRIPT TYPE="text/javascript">
 <&| /elements/onload.js &>
@@ -306,6 +298,26 @@ Example:
       el.attachEvent('onchange', clear_coords);
     }
   }
+  function clear_censustract() {
+    // if the user manually edits the census tract, clear the 'hard' census
+    // tract field so that we can re-verify and present a confirmation popup 
+
+    // get the ID of the hidden censustract field
+    var censustract_id = this.id.replace('enter_', '');
+    var el = document.getElementById(censustract_id);
+    if (el) {
+      el.value = '';
+    }
+  }
+  var el = document.getElementById('<%$pre%>enter_censustract');
+  if (el) {
+    if ( el.addEventListener ) {
+      el.addEventListener('change', clear_censustract);
+    } else if ( el.attachEvent ) {
+      el.attachEvent('onchange', clear_censustract);
+    }
+  }
+
 </&>
 </SCRIPT>
 
diff --git a/httemplate/elements/standardize_locations.js b/httemplate/elements/standardize_locations.js
index a4f13d78d..ff7183b26 100644
--- a/httemplate/elements/standardize_locations.js
+++ b/httemplate/elements/standardize_locations.js
@@ -279,10 +279,7 @@ function setselect(el, value) {
 function confirm_censustract() {
 %   if ( FS::Conf->new->exists('cust_main-require_censustract') ) {
   var form = document.<% $formname %>;
-  // this is the existing/confirmed censustract, not the manually entered one
-  if ( form.elements['censustract'].value == '' ||
-       form.elements['censustract'].value != 
-          form.elements['enter_censustract'].value ) {
+  if ( form.elements['censustract'].value == '' ) {
     var address_info = form_address_info();
     address_info['latitude']  = form.elements['latitude'].value;
     address_info['longitude'] = form.elements['longitude'].value;
diff --git a/httemplate/misc/xmlhttp-address_standardize.html b/httemplate/misc/xmlhttp-address_standardize.html
index 618265364..d0255a02a 100644
--- a/httemplate/misc/xmlhttp-address_standardize.html
+++ b/httemplate/misc/xmlhttp-address_standardize.html
@@ -43,6 +43,10 @@ foreach my $pre ( @prefixes ) {
     last if !$all_same;
   }
 
+  $all_same = 0 if ( length( $old{$pre.'censustract'} ) > 0 &&
+                     length( $new{$pre.'censustract'} ) > 0 &&
+                     $old{$pre.'censustract'} ne $new{$pre.'censustract'} );
+
   $all_same = 0 if $new{$pre.'error'};
 }