support for cch fixed format

6 files changed, 124 insertions, 19 deletions

diff --git a/FS/FS/Misc.pm b/FS/FS/Misc.pm
index e254b51fc..d79f86e0c 100644
--- a/FS/FS/Misc.pm
+++ b/FS/FS/Misc.pm
@@ -7,6 +7,7 @@ use Carp;
 use Data::Dumper;
 use IPC::Run qw( run timeout );   # for _pslatex
 use IPC::Run3; # for do_print... should just use IPC::Run i guess
+use File::Temp;
 #do NOT depend on any FS:: modules here, causes weird (sometimes unreproducable
 #until on client machine) dependancy loops.  put them in FS::Misc::Something
 #instead
@@ -16,6 +17,7 @@ use IPC::Run3; # for do_print... should just use IPC::Run i guess
                  states_hash counties state_label
                  card_types
                  generate_ps generate_pdf do_print
+                 csv_from_fixed
                );
 
 $DEBUG = 0;
@@ -774,6 +776,65 @@ sub do_print {
 
 }
 
+=item csv_from_fixed, FILEREF COUNTREF, [ LENGTH_LISTREF, [ CALLBACKS_LISTREF ] ]
+
+Converts the filehandle referenced by FILEREF from fixed length record
+lines to a CSV file according to the lengths specified in LENGTH_LISTREF.
+The CALLBACKS_LISTREF refers to a correpsonding list of coderefs.  Each
+should return the value to be substituted in place of its single argument.
+
+Returns false on success or an error if one occurs.
+
+=cut
+
+sub csv_from_fixed {
+  my( $fhref, $countref, $lengths, $callbacks) = @_;
+
+  eval { require Text::CSV_XS; };
+  return $@ if $@;
+
+  my $ofh = $$fhref;
+  my $unpacker = new Text::CSV_XS;
+  my $total = 0;
+  my $template = join('', map {$total += $_; "A$_"} @$lengths) if $lengths;
+
+  my $dir = "%%%FREESIDE_CACHE%%%/cache.$FS::UID::datasrc";
+  my $fh = new File::Temp( TEMPLATE => "CODE.csv.XXXXXXXX",
+                           DIR      => $dir,
+                           UNLINK   => 0,
+                         ) or return "can't open temp file: $!\n"
+    if $template;
+
+  while ( defined(my $line=<$ofh>) ) {
+    $$countref++;
+    if ( $template ) {
+      my $column = 0;
+
+      chomp $line;
+      return "unexpected input at line $$countref: $line".
+             " -- expected $total but received ". length($line)
+        unless length($line) == $total;
+
+      $unpacker->combine( map { my $i = $column++;
+                                defined( $callbacks->[$i] )
+                                  ? &{ $callbacks->[$i] }( $_ )
+                                  : $_
+                              } unpack( $template, $line )
+                        )
+        or return "invalid data for CSV: ". $unpacker->error_input;
+
+      print $fh $unpacker->string(), "\n"
+        or return "can't write temp file: $!\n";
+    }
+  }
+
+  if ( $template ) { close $$fhref; $$fhref = $fh }
+
+  seek $$fhref, 0, 0;
+  '';
+}
+
+
 =back
 
 =head1 BUGS
diff --git a/FS/FS/Schema.pm b/FS/FS/Schema.pm
index 8961f33ec..26900b085 100644
--- a/FS/FS/Schema.pm
+++ b/FS/FS/Schema.pm
@@ -1110,7 +1110,7 @@ sub tables_hashref {
         'taxproductnum', 'serial',      '',        '', '', '',
         'data_vendor',   'varchar', 'NULL',   $char_d, '', '', 
         'taxproduct',    'varchar',     '',   $char_d, '', '', 
-        'description',   'varchar',     '', 2*$char_d, '', '', 
+        'description',   'varchar',     '', 3*$char_d, '', '', 
       ],
       'primary_key' => 'taxproductnum',
       'unique'      => [ [ 'data_vendor', 'taxproduct' ] ],
diff --git a/FS/FS/cust_tax_location.pm b/FS/FS/cust_tax_location.pm
index 66d32a5a7..cd24cc8a8 100644
--- a/FS/FS/cust_tax_location.pm
+++ b/FS/FS/cust_tax_location.pm
@@ -3,6 +3,7 @@ package FS::cust_tax_location;
 use strict;
 use vars qw( @ISA );
 use FS::Record qw( qsearch qsearchs dbh );
+use FS::Misc qw ( csv_from_fixed );
 
 @ISA = qw(FS::Record);
 
@@ -135,12 +136,19 @@ sub batch_import {
   my @fields;
   my $hook;
 
+  my @column_lengths = ();
+  my @column_callbacks = ();
+  if ( $format eq 'cch-fixed' || $format eq 'cch-fixed-update' ) {
+    $format =~ s/-fixed//;
+    push @column_lengths, qw( 5 2 4 4 10 1 );
+    push @column_lengths, 1 if $format eq 'cch-update';
+  }
+
   my $line;
   my ( $count, $last, $min_sec ) = (0, time, 5); #progressbar
-  if ( $job ) {
-    $count++
-      while ( defined($line=<$fh>) );
-    seek $fh, 0, 0;
+  if ( $job || scalar(@column_callbacks) ) {
+    my $error = csv_from_fixed(\$fh, \$count, \@column_lengths);
+    return $error if $error;
   }
 
   if ( $format eq 'cch' || $format eq 'cch-update' ) {
diff --git a/FS/FS/part_pkg_taxrate.pm b/FS/FS/part_pkg_taxrate.pm
index 6267d7a44..15636217b 100644
--- a/FS/FS/part_pkg_taxrate.pm
+++ b/FS/FS/part_pkg_taxrate.pm
@@ -6,6 +6,7 @@ use Date::Parse;
 use FS::UID qw(dbh);
 use FS::Record qw( qsearch qsearchs );
 use FS::part_pkg_taxproduct;
+use FS::Misc qw(csv_from_fixed);
 
 @ISA = qw(FS::Record);
 
@@ -175,12 +176,25 @@ sub batch_import {
   my @fields;
   my $hook;
 
+  my @column_lengths = ();
+  my @column_callbacks = ();
+  if ( $format eq 'cch-fixed' || $format eq 'cch-fixed-update' ) {
+    $format =~ s/-fixed//;
+    my $date_format = sub { my $r='';
+                            /^(\d{4})(\d{2})(\d{2})$/ && ($r="$1/$2/$3");
+                            $r;
+                          };
+    $column_callbacks[16] = $date_format;
+    push @column_lengths, qw( 28 25 2 1 10 4 30 3 100 2 2 2 2 1 2 2 8 1 );
+    push @column_lengths, 1 if $format eq 'cch-update';
+  }
+
   my $line;
   my ( $count, $last, $min_sec ) = (0, time, 5); #progressbar
-  if ( $job ) {
-    $count++
-      while ( defined($line=<$fh>) );
-    seek $fh, 0, 0;
+  if ( $job || scalar(@column_callbacks) ) {
+    my $error =
+      csv_from_fixed(\$fh, \$count, \@column_lengths, \@column_callbacks);
+    return $error if $error;
   }
 
   if ( $format eq 'cch' ||  $format eq 'cch-update' ) {
diff --git a/FS/FS/tax_class.pm b/FS/FS/tax_class.pm
index 2fa9fb0f8..8ce70f5c3 100644
--- a/FS/FS/tax_class.pm
+++ b/FS/FS/tax_class.pm
@@ -4,6 +4,7 @@ use strict;
 use vars qw( @ISA );
 use FS::UID qw(dbh);
 use FS::Record qw( qsearch qsearchs );
+use FS::Misc qw( csv_from_fixed );
 
 @ISA = qw(FS::Record);
 
@@ -147,12 +148,19 @@ sub batch_import {
   my $imported = 0;
   my $dbh = dbh;
 
+  my @column_lengths = ();
+  my @column_callbacks = ();
+  if ( $format eq 'cch-fixed' || $format eq 'cch-fixed-update' ) {
+    $format =~ s/-fixed//;
+    push @column_lengths, qw( 8 10 3 2 2 10 100 );
+    push @column_lengths, 1 if $format eq 'cch-update';
+  }
+
   my $line;
   my ( $count, $last, $min_sec ) = (0, time, 5); #progressbar
-  if ( $job ) {
-    $count++
-      while ( defined($line=<$fh>) );
-    seek $fh, 0, 0;
+  if ( $job || scalar(@column_callbacks) ) {
+    my $error = csv_from_fixed(\$fh, \$count, \@column_lengths);
+    return $error if $error;
   }
 
   if ( $format eq 'cch' || $format eq 'cch-update' ) {
diff --git a/FS/FS/tax_rate.pm b/FS/FS/tax_rate.pm
index bfb9c8c32..f45d0143e 100644
--- a/FS/FS/tax_rate.pm
+++ b/FS/FS/tax_rate.pm
@@ -13,6 +13,7 @@ use FS::cust_bill_pkg;
 use FS::cust_tax_location;
 use FS::part_pkg_taxrate;
 use FS::cust_main;
+use FS::Misc qw( csv_from_fixed );
 
 @ISA = qw( FS::Record );
 
@@ -503,12 +504,25 @@ sub batch_import {
   my @fields;
   my $hook;
 
+  my @column_lengths = ();
+  my @column_callbacks = ();
+  if ( $format eq 'cch-fixed' || $format eq 'cch-fixed-update' ) {
+    $format =~ s/-fixed//;
+    my $date_format = sub { my $r='';
+                            /^(\d{4})(\d{2})(\d{2})$/ && ($r="$1/$2/$3");
+                            $r;
+                          };
+    $column_callbacks[8] = $date_format;
+    push @column_lengths, qw( 10 1 1 8 8 5 8 8 8 1 2 2 30 8 8 10 2 8 2 1 2 2 );
+    push @column_lengths, 1 if $format eq 'cch-update';
+  }
+  
   my $line;
   my ( $count, $last, $min_sec ) = (0, time, 5); #progressbar
-  if ( $job ) {
-    $count++
-      while ( defined($line=<$fh>) );
-    seek $fh, 0, 0;
+  if ( $job || scalar(@column_callbacks) ) {
+    my $error =
+      csv_from_fixed(\$fh, \$count, \@column_lengths, \@column_callbacks);
+    return $error if $error;
   }
   $count *=2;
 
@@ -745,7 +759,7 @@ sub process_batch {
 
   my (%files) = map { /^(\w+):([\.\w]+)$/ ? ($1,$2):() } split /,/, $files;
 
-  if ($format eq 'cch') {
+  if ($format eq 'cch' || $format eq 'cch-fixed') {
 
     my $oldAutoCommit = $FS::UID::AutoCommit;
     local $FS::UID::AutoCommit = 0;
@@ -779,7 +793,7 @@ sub process_batch {
       $dbh->commit or die $dbh->errstr if $oldAutoCommit;
     }
 
-  }elsif ($format eq 'cch-update') {
+  }elsif ($format eq 'cch-update' || $format eq 'cch-fixed-update') {
 
     my $oldAutoCommit = $FS::UID::AutoCommit;
     local $FS::UID::AutoCommit = 0;