package web import from CSV/XLS, RT#9529

5 files changed, 457 insertions, 30 deletions

diff --git a/FS/FS/Mason.pm b/FS/FS/Mason.pm
index 2a4b42f..f5d7c85 100644
--- a/FS/FS/Mason.pm
+++ b/FS/FS/Mason.pm
@@ -3,6 +3,7 @@ package FS::Mason;
 use strict;
 use vars qw( @ISA @EXPORT_OK $addl_handler_use );
 use Exporter;
+use Carp;
 use File::Slurp qw( slurp );
 use HTML::Mason 1.27; #http://www.masonhq.com/?ApacheModPerl2Redirect
 use HTML::Mason::Interp;
@@ -146,6 +147,7 @@ if ( -e $addl_handler_use_file ) {
   use FS::cust_location;
   use FS::cust_pay;
   use FS::cust_pkg;
+  use FS::cust_pkg::Import;
   use FS::part_pkg_taxclass;
   use FS::cust_pkg_reason;
   use FS::cust_refund;
@@ -361,6 +363,7 @@ if ( -e $addl_handler_use_file ) {
   
   sub include {
     use vars qw($m);
+    #carp #should just switch to <& &> syntax
     $m->scomp(@_);
   }
 
diff --git a/FS/FS/Record.pm b/FS/FS/Record.pm
index bc075dd..758e0f9 100644
--- a/FS/FS/Record.pm
+++ b/FS/FS/Record.pm
@@ -1611,6 +1611,8 @@ Class method for batch imports.  Available params:
 
 =item table
 
+=item format - usual way to specify import, with this format string selecting data from the formats and format_* info hashes
+
 =item formats
 
 =item format_types
@@ -1623,6 +1625,10 @@ Class method for batch imports.  Available params:
 
 =item format_row_callbacks
 
+=item fields - Alternate way to specify import, specifying import fields directly as a listref
+
+=item postinsert_callback
+
 =item params
 
 =item job
@@ -1635,8 +1641,6 @@ FS::queue object, will be updated with progress
 
 csv, xls or fixedlength
 
-=item format
-
 =item empty_ok
 
 =back
@@ -1647,21 +1651,64 @@ sub batch_import {
   my $param = shift;
 
   warn "$me batch_import call with params: \n". Dumper($param)
-  ;#  if $DEBUG;
+    if $DEBUG;
 
   my $table   = $param->{table};
-  my $formats = $param->{formats};
 
   my $job     = $param->{job};
   my $file    = $param->{file};
-  my $format  = $param->{'format'};
   my $params  = $param->{params} || {};
 
-  die "unknown format $format" unless exists $formats->{ $format };
+  my( $type, $header, $sep_char, $fixedlength_format, $row_callback, @fields );
+  my $postinsert_callback = '';
+  if ( $param->{'format'} ) {
+
+    my $format  = $param->{'format'};
+    my $formats = $param->{formats};
+    die "unknown format $format" unless exists $formats->{ $format };
+
+    $type = $param->{'format_types'}
+            ? $param->{'format_types'}{ $format }
+            : $param->{type} || 'csv';
+
+
+    $header = $param->{'format_headers'}
+               ? $param->{'format_headers'}{ $param->{'format'} }
+               : 0;
+
+    $sep_char = $param->{'format_sep_chars'}
+                  ? $param->{'format_sep_chars'}{ $param->{'format'} }
+                  : ',';
+
+    $fixedlength_format =
+      $param->{'format_fixedlength_formats'}
+        ? $param->{'format_fixedlength_formats'}{ $param->{'format'} }
+        : '';
+
+    $row_callback =
+      $param->{'format_row_callbacks'}
+        ? $param->{'format_row_callbacks'}{ $param->{'format'} }
+        : '';
+
+    @fields = @{ $formats->{ $format } };
+
+  } elsif ( $param->{'fields'} ) {
+
+    $type = ''; #infer from filename
+    $header = 0;
+    $sep_char = ',';
+    $fixedlength_format = '';
+    $row_callback = '';
+    @fields = @{ $param->{'fields'} };
 
-  my $type = $param->{'format_types'}
-             ? $param->{'format_types'}{ $format }
-             : $param->{type} || 'csv';
+    $postinsert_callback = $param->{'postinsert_callback'}
+      if $param->{'postinsert_callback'}
+
+  } else {
+    die "neither format nor fields specified";
+  }
+
+  #my $file    = $param->{file};
 
   unless ( $type ) {
     if ( $file =~ /\.(\w+)$/i ) {
@@ -1675,25 +1722,6 @@ sub batch_import {
       if $param->{'default_csv'} && $type ne 'xls';
   }
 
-  my $header = $param->{'format_headers'}
-                 ? $param->{'format_headers'}{ $param->{'format'} }
-                 : 0;
-
-  my $sep_char = $param->{'format_sep_chars'}
-                   ? $param->{'format_sep_chars'}{ $param->{'format'} }
-                   : ',';
-
-  my $fixedlength_format =
-    $param->{'format_fixedlength_formats'}
-      ? $param->{'format_fixedlength_formats'}{ $param->{'format'} }
-      : '';
-
-  my $row_callback =
-    $param->{'format_row_callbacks'}
-      ? $param->{'format_row_callbacks'}{ $param->{'format'} }
-      : '';
-
-  my @fields = @{ $formats->{ $format } };
 
   my $row = 0;
   my $count;
@@ -1757,6 +1785,7 @@ sub batch_import {
   local $FS::UID::AutoCommit = 0;
   my $dbh = dbh;
 
+  #my $params  = $param->{params} || {};
   if ( $param->{'batch_namecol'} && $param->{'batch_namevalue'} ) {
     my $batch_col   = $param->{'batch_keycol'};
 
@@ -1774,7 +1803,8 @@ sub batch_import {
 
     $params->{ $batch_col } = $batch_value;
   }
-  
+
+  #my $job     = $param->{job};
   my $line;
   my $imported = 0;
   my( $last, $min_sec ) = ( time, 5 ); #progressbar foo
@@ -1832,6 +1862,7 @@ sub batch_import {
 
     }
 
+    #my $table   = $param->{table};
     my $class = "FS::$table";
 
     my $record = $class->new( \%hash );
@@ -1855,6 +1886,15 @@ sub batch_import {
     $row++;
     $imported++;
 
+    if ( $postinsert_callback ) {
+      my $error = &{$postinsert_callback}($record, $param);
+      if ( $error ) {
+        $dbh->rollback if $oldAutoCommit;
+        return "postinsert_callback error". ( $line ? " for $line" : '' ).
+               ": $error";
+      }
+    }
+
     if ( $job && time - $min_sec > $last ) { #progress bar
       $job->update_statustext( int(100 * $imported / $count) );
       $last = time;
diff --git a/FS/FS/Schema.pm b/FS/FS/Schema.pm
index d7d5a04..dc8f2f3 100644
--- a/FS/FS/Schema.pm
+++ b/FS/FS/Schema.pm
@@ -1289,6 +1289,7 @@ sub tables_hashref {
         'pkgnum',           'serial',     '', '', '', '', 
         'custnum',             'int',     '', '', '', '', 
         'pkgpart',             'int',     '', '', '', '', 
+        'pkgbatch',        'varchar', 'NULL', $char_d, '', '',
         'locationnum',         'int', 'NULL', '', '', '',
         'otaker',          'varchar', 'NULL', 32, '', '', 
         'usernum',             'int', 'NULL', '', '', '',
@@ -1310,7 +1311,8 @@ sub tables_hashref {
       ],
       'primary_key' => 'pkgnum',
       'unique' => [],
-      'index' => [ ['custnum'], ['pkgpart'], [ 'locationnum' ], [ 'usernum' ],
+      'index' => [ ['custnum'], ['pkgpart'], [ 'pkgbatch' ], [ 'locationnum' ],
+                   [ 'usernum' ],
                    [ 'start_date' ], ['setup'], ['last_bill'], ['bill'],
                    ['susp'], ['adjourn'], ['expire'], ['cancel'],
                    ['change_date'],
diff --git a/FS/FS/cust_pkg.pm b/FS/FS/cust_pkg.pm
index 0f9a611..c3ee4e4 100644
--- a/FS/FS/cust_pkg.pm
+++ b/FS/FS/cust_pkg.pm
@@ -2624,6 +2624,15 @@ sub search {
   }
 
   ##
+  # custbatch
+  ##
+
+  if ( $params->{'pkgbatch'} =~ /^([\w\/\-\:\.]+)$/ and $1 ) {
+    push @where,
+      "cust_pkg.pkgbatch = '$1'";
+  }
+
+  ##
   # parse status
   ##
 
diff --git a/FS/FS/cust_pkg/Import.pm b/FS/FS/cust_pkg/Import.pm
new file mode 100644
index 0000000..7a4b9d5
--- /dev/null
+++ b/FS/FS/cust_pkg/Import.pm
@@ -0,0 +1,373 @@
+package FS::cust_pkg::Import;
+
+use strict;
+use vars qw( $DEBUG ); #$conf );
+use Storable qw(thaw);
+use Data::Dumper;
+use MIME::Base64;
+use FS::Misc::DateTime qw( parse_datetime );
+use FS::Record qw( qsearchs );
+use FS::cust_pkg;
+use FS::cust_main;
+use FS::svc_acct;
+use FS::svc_external;
+use FS::svc_phone;
+
+$DEBUG = 0;
+
+#install_callback FS::UID sub {
+#  $conf = new FS::Conf;
+#};
+
+=head1 NAME
+
+FS::cust_pkg::Import - Batch customer importing
+
+=head1 SYNOPSIS
+
+  use FS::cust_pkg::Import;
+
+  #import
+  FS::cust_pkg::Import::batch_import( {
+    file      => $file,      #filename
+    type      => $type,      #csv or xls
+    format    => $format,    #extended, extended-plus_company, svc_external,
+                             # or svc_external_svc_phone
+    agentnum  => $agentnum,
+    job       => $job,       #optional job queue job, for progressbar updates
+    pkgbatch  => $pkgbatch, #optional batch unique identifier
+  } );
+  die $error if $error;
+
+  #ajax helper
+  use FS::UI::Web::JSRPC;
+  my $server =
+    new FS::UI::Web::JSRPC 'FS::cust_pkg::Import::process_batch_import', $cgi;
+  print $server->process;
+
+=head1 DESCRIPTION
+
+Batch package importing.
+
+=head1 SUBROUTINES
+
+=item process_batch_import
+
+Load a batch import as a queued JSRPC job
+
+=cut
+
+sub process_batch_import {
+  my $job = shift;
+
+  my $param = thaw(decode_base64(shift));
+  warn Dumper($param) if $DEBUG;
+  
+  my $files = $param->{'uploaded_files'}
+    or die "No files provided.\n";
+
+  my (%files) = map { /^(\w+):([\.\w]+)$/ ? ($1,$2):() } split /,/, $files;
+
+  my $dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc. '/';
+  my $file = $dir. $files{'file'};
+
+  my $type;
+  if ( $file =~ /\.(\w+)$/i ) {
+    $type = lc($1);
+  } else {
+    #or error out???
+    warn "can't parse file type from filename $file; defaulting to CSV";
+    $type = 'csv';
+  }
+
+  my $error =
+    FS::cust_pkg::Import::batch_import( {
+      job      => $job,
+      file     => $file,
+      type     => $type,
+      'params' => { pkgbatch => $param->{pkgbatch} },
+      agentnum => $param->{'agentnum'},
+      'format' => $param->{'format'},
+    } );
+
+  unlink $file;
+
+  die "$error\n" if $error;
+
+}
+
+=item batch_import
+
+=cut
+
+my %formatfields = (
+  'default'      => [],
+  'svc_acct'     => [qw( username _password )],
+  'svc_phone'    => [qw( countrycode phonenum sip_password pin )],
+  'svc_external' => [qw( id title )],
+);
+
+sub _formatfields {
+  \%formatfields;
+}
+
+my %import_options = (
+  'table'         => 'cust_pkg',
+
+  'postinsert_callback' => sub {
+    my( $record, $param ) = @_;
+
+    my $formatfields = _formatfields;
+    foreach my $svc_x ( grep { $_ ne 'default' } keys %$formatfields ) {
+
+      my $ff = $formatfields->{$svc_x};
+
+      if ( grep $param->{"$svc_x.$_"}, @$ff ) {
+        my $svc_x = "FS::$svc_x"->new( {
+          'pkgnum'  => $record->pkgnum,
+          'svcpart' => $record->part_pkg->svcpart($svc_x),
+          map { $_ => $param->{"$svc_x.$_"} } @$ff
+        } );
+        my $error = $svc_x->insert;
+        return $error if $error;
+      }
+
+    }
+
+    return ''; #no error
+
+  },
+);
+
+sub _import_options {
+  \%import_options;
+}
+
+sub batch_import {
+  my $opt = shift;
+
+  my $iopt = _import_options;
+  $opt->{$_} = $iopt->{$_} foreach keys %$iopt;
+
+  my $agentnum  = delete $opt->{agentnum}; # i like closures (delete though?)
+
+  my $format = delete $opt->{'format'};
+  my @fields = ();
+
+  if ( $format =~ /^(.*)-agent_custid$/ ) {
+    $format = $1;
+    @fields = (
+      sub {
+        my( $self, $value ) = @_; # $conf, $param
+        my $cust_main = qsearchs('cust_main', {
+          'agentnum'     => $agentnum,
+          'agent_custid' => $value,
+        });
+        $self->custnum($cust_main->custnum) if $cust_main;
+      },
+    );
+  } else {
+    @fields = ( 'custnum' );
+  }
+
+  push @fields, ( 'pkgpart', 'discountnum' );
+
+  foreach my $field ( 
+    qw( start_date setup bill last_bill susp adjourn cancel expire )
+  ) {
+    push @fields, sub {
+      my( $self, $value ) = @_; # $conf, $param
+      #->$field has undesirable effects
+      $self->set($field, parse_datetime($value) ); #$field closure
+    };
+  }
+
+  my $formatfields = _formatfields();
+
+  die "unknown format $format" unless $formatfields->{$format};
+
+  foreach my $field ( @{ $formatfields->{$format} } ) {
+
+    push @fields, sub {
+      my( $self, $value, $conf, $param ) = @_;
+      $param->{"$format.$field"} = $value;
+    };
+
+  }
+
+  $opt->{'fields'} = \@fields;
+
+  FS::Record::batch_import( $opt );
+
+}
+
+=for comment
+
+    my $billtime = time;
+    my %cust_pkg = ( pkgpart => $pkgpart );
+    my %svc_x = ();
+    foreach my $field ( @fields ) {
+
+      if ( $field =~ /^cust_pkg\.(pkgpart|setup|bill|susp|adjourn|expire|cancel)$/ ) {
+
+        #$cust_pkg{$1} = parse_datetime( shift @$columns );
+        if ( $1 eq 'pkgpart' ) {
+          $cust_pkg{$1} = shift @columns;
+        } elsif ( $1 eq 'setup' ) {
+          $billtime = parse_datetime(shift @columns);
+        } else {
+          $cust_pkg{$1} = parse_datetime( shift @columns );
+        } 
+
+      } elsif ( $field =~ /^svc_acct\.(username|_password)$/ ) {
+
+        $svc_x{$1} = shift @columns;
+
+      } elsif ( $field =~ /^svc_external\.(id|title)$/ ) {
+
+        $svc_x{$1} = shift @columns;
+
+      } elsif ( $field =~ /^svc_phone\.(countrycode|phonenum|sip_password|pin)$/ ) {
+        $svc_x{$1} = shift @columns;
+       
+      } else {
+
+        #refnum interception
+        if ( $field eq 'refnum' && $columns[0] !~ /^\s*(\d+)\s*$/ ) {
+
+          my $referral = $columns[0];
+          my %hash = ( 'referral' => $referral,
+                       'agentnum' => $agentnum,
+                       'disabled' => '',
+                     );
+
+          my $part_referral = qsearchs('part_referral', \%hash )
+                              || new FS::part_referral \%hash;
+
+          unless ( $part_referral->refnum ) {
+            my $error = $part_referral->insert;
+            if ( $error ) {
+              $dbh->rollback if $oldAutoCommit;
+              return "can't auto-insert advertising source: $referral: $error";
+            }
+          }
+
+          $columns[0] = $part_referral->refnum;
+        }
+
+        my $value = shift @columns;
+        $cust_main{$field} = $value if length($value);
+      }
+    }
+
+    $cust_main{'payby'} = 'CARD'
+      if defined $cust_main{'payinfo'}
+      && length  $cust_main{'payinfo'};
+
+    my $invoicing_list = $cust_main{'invoicing_list'}
+                           ? [ delete $cust_main{'invoicing_list'} ]
+                           : [];
+
+    my $cust_main = new FS::cust_main ( \%cust_main );
+
+    use Tie::RefHash;
+    tie my %hash, 'Tie::RefHash'; #this part is important
+
+    if ( $cust_pkg{'pkgpart'} ) {
+      my $cust_pkg = new FS::cust_pkg ( \%cust_pkg );
+
+      my @svc_x = ();
+      my $svcdb = '';
+      if ( $svc_x{'username'} ) {
+        $svcdb = 'svc_acct';
+      } elsif ( $svc_x{'id'} || $svc_x{'title'} ) {
+        $svcdb = 'svc_external';
+      }
+
+      my $svc_phone = '';
+      if ( $svc_x{'countrycode'} || $svc_x{'phonenum'} ) {
+        $svc_phone = FS::svc_phone->new( {
+          map { $_ => delete($svc_x{$_}) }
+              qw( countrycode phonenum sip_password pin)
+        } );
+      }
+
+      if ( $svcdb || $svc_phone ) {
+        my $part_pkg = $cust_pkg->part_pkg;
+	unless ( $part_pkg ) {
+	  $dbh->rollback if $oldAutoCommit;
+	  return "unknown pkgpart: ". $cust_pkg{'pkgpart'};
+	} 
+        if ( $svcdb ) {
+          $svc_x{svcpart} = $part_pkg->svcpart_unique_svcdb( $svcdb );
+          my $class = "FS::$svcdb";
+          push @svc_x, $class->new( \%svc_x );
+        }
+        if ( $svc_phone ) {
+          $svc_phone->svcpart( $part_pkg->svcpart_unique_svcdb('svc_phone') );
+          push @svc_x, $svc_phone;
+        }
+      }
+
+      $hash{$cust_pkg} = \@svc_x;
+    }
+
+    my $error = $cust_main->insert( \%hash, $invoicing_list );
+
+    if ( $error ) {
+      $dbh->rollback if $oldAutoCommit;
+      return "can't insert customer". ( $line ? " for $line" : '' ). ": $error";
+    }
+
+    if ( $format eq 'simple' ) {
+
+      #false laziness w/bill.cgi
+      $error = $cust_main->bill( 'time' => $billtime );
+      if ( $error ) {
+        $dbh->rollback if $oldAutoCommit;
+        return "can't bill customer for $line: $error";
+      }
+  
+      $error = $cust_main->apply_payments_and_credits;
+      if ( $error ) {
+        $dbh->rollback if $oldAutoCommit;
+        return "can't bill customer for $line: $error";
+      }
+
+      $error = $cust_main->collect();
+      if ( $error ) {
+        $dbh->rollback if $oldAutoCommit;
+        return "can't collect customer for $line: $error";
+      }
+
+    }
+
+    $row++;
+
+    if ( $job && time - $min_sec > $last ) { #progress bar
+      $job->update_statustext( int(100 * $row / $count) );
+      $last = time;
+    }
+
+  }
+
+  $dbh->commit or die $dbh->errstr if $oldAutoCommit;;
+
+  return "Empty file!" unless $row;
+
+  ''; #no error
+
+}
+
+=head1 BUGS
+
+Not enough documentation.
+
+=head1 SEE ALSO
+
+L<FS::cust_main>, L<FS::cust_pkg>,
+L<FS::svc_acct>, L<FS::svc_external>, L<FS::svc_phone>
+
+=cut
+
+1;