From: ivan Date: Thu, 14 Aug 2008 11:56:12 +0000 (+0000) Subject: customer import: add progress bar & redirect to a search of the imported customers... X-Git-Tag: freeside_1_7_4rc1~218 X-Git-Url: http://git.freeside.biz/gitweb/?a=commitdiff_plain;h=a9d755c3cee7542d3ca552174af0b7b71693ae58;p=freeside.git customer import: add progress bar & redirect to a search of the imported customers, #3475 --- diff --git a/FS/FS/Schema.pm b/FS/FS/Schema.pm index 956aedd92..1d6bd7c07 100644 --- a/FS/FS/Schema.pm +++ b/FS/FS/Schema.pm @@ -486,6 +486,7 @@ sub tables_hashref { 'custnum', 'serial', '', '', '', '', 'agentnum', 'int', '', '', '', '', 'agent_custid', 'varchar', 'NULL', $char_d, '', '', + 'custbatch', 'varchar', 'NULL', $char_d, '', '', # 'titlenum', 'int', 'NULL', '', '', '', 'last', 'varchar', '', $char_d, '', '', # 'middle', 'varchar', 'NULL', $char_d, '', '', @@ -548,7 +549,7 @@ sub tables_hashref { [ 'county' ], [ 'state' ], [ 'country' ], [ 'zip' ], [ 'ship_last' ], [ 'ship_company' ], [ 'payby' ], [ 'paydate' ], - + [ 'agentnum' ], [ 'custbatch' ], ], }, diff --git a/FS/FS/UID.pm b/FS/FS/UID.pm index 8dd928ec7..0f11c2b53 100644 --- a/FS/FS/UID.pm +++ b/FS/FS/UID.pm @@ -3,7 +3,7 @@ package FS::UID; use strict; use vars qw( @ISA @EXPORT_OK $cgi $dbh $freeside_uid $user - $conf_dir $secrets $datasrc $db_user $db_pass %callback @callback + $conf_dir $cache_dir $secrets $datasrc $db_user $db_pass %callback @callback $driver_name $AutoCommit ); use subs qw( @@ -22,6 +22,7 @@ use FS::CurrentUser; $freeside_uid = scalar(getpwnam('freeside')); $conf_dir = "%%%FREESIDE_CONF%%%/"; +$cache_dir = "%%%FREESIDE_CACHE%%%"; $AutoCommit = 1; #ours, not DBI diff --git a/FS/FS/cust_main.pm b/FS/FS/cust_main.pm index 391a2cb31..f0c1deacd 100644 --- a/FS/FS/cust_main.pm +++ b/FS/FS/cust_main.pm @@ -1211,6 +1211,7 @@ sub check { || $self->ut_number('agentnum') || $self->ut_textn('agent_custid') || $self->ut_number('refnum') + || $self->ut_textn('custbatch') || $self->ut_name('last') || $self->ut_name('first') || $self->ut_snumbern('birthdate') @@ -4809,6 +4810,15 @@ sub search_sql { push @where, map { s/current_balance/$balance_sql/; $_ } @{ $params->{'current_balance'} }; + ## + # custbatch + ## + + if ( $params->{'custbatch'} =~ /^([\w\/\-\:\.]+)$/ and $1 ) { + push @where, + "cust_main.custbatch = '$1'"; + } + ## # setup queries, subs, etc. for the search ## @@ -5470,6 +5480,58 @@ sub append_fuzzyfiles { 1; } +=item process_batch_import + +Load a batch import as a queued JSRPC job + +=cut + +use Storable qw(thaw); +use Data::Dumper; +use MIME::Base64; +sub process_batch_import { + my $job = shift; + + my $param = thaw(decode_base64(shift)); + warn Dumper($param) if $DEBUG; + + my $files = $param->{'uploaded_files'} + or die "No files provided.\n"; + + my (%files) = map { /^(\w+):([\.\w]+)$/ ? ($1,$2):() } split /,/, $files; + + my $dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc. '/'; + my $file = $dir. $files{'file'}; + + my $type; + if ( $file =~ /\.(\w+)$/i ) { + $type = lc($1); + } else { + #or error out??? + warn "can't parse file type from filename $file; defaulting to CSV"; + $type = 'csv'; + } + + my $error = + FS::cust_main::batch_import( { + job => $job, + file => $file, + type => $type, + custbatch => $param->{custbatch}, + agentnum => $param->{'agentnum'}, + refnum => $param->{'refnum'}, + pkgpart => $param->{'pkgpart'}, + #'fields' => [qw( cust_pkg.setup dayphone first last address1 address2 + # city state zip comments )], + 'format' => $param->{'format'}, + } ); + + unlink $file; + + die "$error\n" if $error; + +} + =item batch_import =cut @@ -5478,14 +5540,18 @@ sub append_fuzzyfiles { sub batch_import { my $param = shift; - my $fh = $param->{filehandle}; - my $type = $param->{type} || 'csv'; + my $job = $param->{job}; + + my $filename = $param->{file}; + my $type = $param->{type} || 'csv'; + + my $custbatch = $param->{custbatch}; - my $agentnum = $param->{agentnum}; - my $refnum = $param->{refnum}; - my $pkgpart = $param->{pkgpart}; + my $agentnum = $param->{agentnum}; + my $refnum = $param->{refnum}; + my $pkgpart = $param->{pkgpart}; - my $format = $param->{'format'}; + my $format = $param->{'format'}; my @fields; my $payby; @@ -5521,28 +5587,30 @@ sub batch_import { die "unknown format $format"; } + my $count; my $parser; - my $spoolfile = ''; + my @buffer = (); if ( $type eq 'csv' ) { + eval "use Text::CSV_XS;"; die $@ if $@; + $parser = new Text::CSV_XS; + + @buffer = split(/\r?\n/, slurp($filename) ); + $count = scalar(@buffer); + } elsif ( $type eq 'xls' ) { eval "use Spreadsheet::ParseExcel;"; die $@ if $@; - ( my $spool_fh, $spoolfile ) = - tempfile('cust_main-batch_import-XXXXXXXXXXXX', - DIR => '%%%FREESIDE_CACHE%%%', - SUFFIX => '.xls', - ); - print $spool_fh slurp($fh); - close $spool_fh or die $!; - - my $excel = new Spreadsheet::ParseExcel::Workbook->Parse($spoolfile); + my $excel = new Spreadsheet::ParseExcel::Workbook->Parse($filename); $parser = $excel->{Worksheet}[0]; #first sheet + $count = $parser->{MaxRow} || $parser->{MinRow}; + $count++; + } else { die "Unknown file type $type\n"; } @@ -5562,12 +5630,14 @@ sub batch_import { my $line; my $row = 0; + my( $last, $min_sec ) = ( time, 5 ); #progressbar foo while (1) { my @columns = (); if ( $type eq 'csv' ) { - last unless defined($line=<$fh>); + last unless scalar(@buffer); + $line = shift(@buffer); $parser->parse($line) or do { $dbh->rollback if $oldAutoCommit; @@ -5592,11 +5662,12 @@ sub batch_import { #warn join('-',@columns); my %cust_main = ( - agentnum => $agentnum, - refnum => $refnum, - country => $conf->config('countrydefault') || 'US', - payby => $payby, #default - paydate => '12/2037', #default + custbatch => $custbatch, + agentnum => $agentnum, + refnum => $refnum, + country => $conf->config('countrydefault') || 'US', + payby => $payby, #default + paydate => '12/2037', #default ); my $billtime = time; my %cust_pkg = ( pkgpart => $pkgpart ); @@ -5648,7 +5719,9 @@ sub batch_import { } } - $cust_main{'payby'} = 'CARD' if length($cust_main{'payinfo'}); + $cust_main{'payby'} = 'CARD' + if defined $cust_main{'payinfo'} + && length $cust_main{'payinfo'}; my $invoicing_list = $cust_main{'invoicing_list'} ? [ delete $cust_main{'invoicing_list'} ] @@ -5680,7 +5753,7 @@ sub batch_import { if ( $error ) { $dbh->rollback if $oldAutoCommit; - return "can't insert customer ". ( $line ? "for $line" : '' ). ": $error"; + return "can't insert customer". ( $line ? " for $line" : '' ). ": $error"; } if ( $format eq 'simple' ) { @@ -5707,11 +5780,15 @@ sub batch_import { } $row++; - } - $dbh->commit or die $dbh->errstr if $oldAutoCommit; + if ( $job && time - $min_sec > $last ) { #progress bar + $job->update_statustext( int(100 * $row / $count) ); + $last = time; + } + + } - unlink($spoolfile) if $spoolfile; + $dbh->commit or die $dbh->errstr if $oldAutoCommit;; return "Empty file!" unless $row; diff --git a/httemplate/elements/file-upload.html b/httemplate/elements/file-upload.html new file mode 100644 index 000000000..023bffb1e --- /dev/null +++ b/httemplate/elements/file-upload.html @@ -0,0 +1,74 @@ + + + + + + +% foreach (@field) { + + <% shift @label %> + + +% } + +
+ Debugging: 

+

+
+<%init>
+
+my %param = @_;
+
+my @label = ref($param{'label'}) ? @{$param{'label'}} : ($param{'label'});
+my @field = ref($param{'field'}) ? @{$param{'field'}} : ($param{'field'});
+
+
diff --git a/httemplate/elements/form-file_upload.html b/httemplate/elements/form-file_upload.html
new file mode 100644
index 000000000..f398ba42d
--- /dev/null
+++ b/httemplate/elements/form-file_upload.html
@@ -0,0 +1,93 @@
+<%doc>
+
+Example:
+
+  <% include( '/elements/form-file_upload.html',
+
+                'name'      => 'form_name',
+                'action'    => 'process/target.cgi', #progress-init target
+                'fields'    => [ 'other', 'form', 'fields' ],
+                'num_files' => 1, #or more
+
+                'url' => $url
+                #OR
+                'message' => 'Message',
+
+                #optional
+                'key' => 'unique_key', #for using more than once on a page
+            )
+
+% #... 
+
+% # num_files=>1
+  include( '/elements/file-upload.html',
+             'field'    => 'element',
+             'label'    => 'Label',
+         )
+
+% # OR
+
+% # num_files=>2 # or more
+  include( '/elements/file-upload.html',
+             'field'    => [ 'element', 'element2', ], #etc.
+             'label'    => [ 'Label',   'Label2',   ], #etc.
+         )
+
+
+%>
+
+
+
+<% include( '/elements/progress-init.html',
+              $opt{name},
+              $opt{fields},
+              $opt{action},
+              $msg_or_url,
+              $opt{key},
+          )
+%>
+
+
+
+

+
+

+
+<%init>
+
+#my( $formname, $fields, $action, $url_or_message, $key ) = @_;
+my %opt = ref($_[0]) ? %{ $_[0] } : @_;
+
+my $key = exists $opt{key} ? $opt{key} : '';
+
+push @{ $opt{fields} }, 'uploaded_files';
+
+my $msg_or_url = $opt{message}
+                   ? { 'message' => $opt{message},
+                       'url'     => $opt{url},
+                     }
+                   : $opt{url};
+
+
diff --git a/httemplate/elements/progress-init.html b/httemplate/elements/progress-init.html
index 2cde86f5b..194fc7480 100644
--- a/httemplate/elements/progress-init.html
+++ b/httemplate/elements/progress-init.html
@@ -73,8 +73,9 @@ $key = '' unless defined $key;
 
 my $url_or_message_link;
 if ( ref($url_or_message) ) { #its a message or something
-  $url_or_message_link =
-    'message='. uri_escape( $url_or_message->{'message'} )
+  $url_or_message_link = 'message='. uri_escape( $url_or_message->{'message'} );
+  $url_or_message_link .= ';url='.   uri_escape( $url_or_message->{'url'} )
+    if $url_or_message->{'url'};
 } else {
   $url_or_message_link = "url=$url_or_message";
 }
diff --git a/httemplate/elements/progress-popup.html b/httemplate/elements/progress-popup.html
index cda704a12..0bd71ff4a 100644
--- a/httemplate/elements/progress-popup.html
+++ b/httemplate/elements/progress-popup.html
@@ -41,15 +41,24 @@ function updateStatus( status_statustext ) {
     //jsrsExecute( '<%$p%>elements/jsrsServer.html', updateStatus, 'job_status', '<% $jobnum %>' );
     job_status( '<% $jobnum %>', updateStatus );
   } else if ( status.indexOf('complete') > -1 ) {
-% if ( $message ) { 
+% if ( $message ) {
+%
+%   my $onClick = $url
+%                   ? "window.top.location.href = \\'$url\\';"
+%                   : 'parent.nd(1);';
 
     document.getElementById("progress_message").innerHTML = "<% $message %>";
     document.getElementById("progress_bar").innerHTML = '';
-    document.getElementById("progress_percent").innerHTML = '';
+    document.getElementById("progress_percent").innerHTML =
+      '';
     document.getElementById("progress_jobnum").innerHTML = '';
-    if ( parent.document.<%$formname%>.submit.disabled == true ) {
-      parent.document.<%$formname%>.submit.disabled=false;
-    }
+
+%   unless ( $url ) {
+        if ( parent.document.<%$formname%>.submit.disabled == true ) {
+          parent.document.<%$formname%>.submit.disabled=false;
+        }
+%   }
+
 % } elsif ( $url ) { 
 
     window.top.location.href = '<% $url %>';
diff --git a/httemplate/misc/cust_main-import.cgi b/httemplate/misc/cust_main-import.cgi
index e1580e5a1..31bf1ee62 100644
--- a/httemplate/misc/cust_main-import.cgi
+++ b/httemplate/misc/cust_main-import.cgi
@@ -3,31 +3,44 @@
 Import a file containing customer records.
 


 
-
+<% include( '/elements/form-file_upload.html',
+              'name'      => 'CustomerImportForm',
+              'action'    => 'process/cust_main-import.cgi',
+              'num_files' => 1,
+              'fields'    => [ 'agentnum', 'custbatch', 'format' ],
+              'message'   => 'Customer import successful',
+              'url'       => $p."search/cust_main.html?custbatch=$custbatch",
+          )
+%>
 
 <% &ntable("#cccccc", 2) %>
 
-<% include('/elements/tr-select-agent.html', '', #$agentnum,
-              'label'       => "Agent",
-              'empty_label' => 'Select agent',
-           )
-%>
+  <% include('/elements/tr-select-agent.html', '', #$agentnum,
+                'label'       => "Agent",
+                'empty_label' => 'Select agent',
+             )
+  %>
+
+  
+
+  
+    Format
+    
+      
+    
+  
+
+  <% include( '/elements/file-upload.html',
+                'field' => 'file',
+                'label' => 'Filename',
+            )
+  %>
 
-
-  Format
-  
-    
-  
-
 
-
-  Filename
-  
-
 % #include('/elements/tr-select-part_referral.html')
 %
 
@@ -48,7 +61,15 @@ Import a file containing customer records.
 
 -->
 
-
+  
+    
+      
+    
+  
 
 
 
@@ -108,4 +129,6 @@ my $req = qq!*!;
 die "access denied"
   unless $FS::CurrentUser::CurrentUser->access_right('Import');
 
+my $custbatch = time2str('webimport-%Y/%m/%d-%T'. "-$$-". rand() * 2**32, time);
+
 
diff --git a/httemplate/misc/file-upload.html b/httemplate/misc/file-upload.html
new file mode 100644
index 000000000..469274c69
--- /dev/null
+++ b/httemplate/misc/file-upload.html
@@ -0,0 +1,53 @@
+<% include('/elements/header-minimal.html', 'File Upload') %>
+% if ($error) {
+Error: <% $error %>
+% }else{
+File Upload Successful <% join(',', @filenames) %>;
+% }
+<% include('/elements/footer.html') %>
+<%init>
+
+die "access denied"
+  unless $FS::CurrentUser::CurrentUser->access_right('Import'); #?
+
+my @filenames = ();
+my $error = '';     # could be extended to the access control
+
+$cgi->param('upload_fields') =~ /^([,\w]+)$/
+  or $error = "invalid upload_fields";
+my $fields = $1;
+
+my $dir = $FS::UID::cache_dir. "/cache.". $FS::UID::datasrc;
+
+foreach my $field (split /,/, $fields) {
+  next if $error;
+
+  my $fh = $cgi->upload($field)
+    or $error = "No valid file was provided.";
+
+  my $suffix = '';
+  if ( $cgi->param($field) =~ /(\.\w+)$/i ) {
+    $suffix = lc($1);
+  }
+
+  my $sh = new File::Temp( TEMPLATE => 'upload.XXXXXXXX',
+                           SUFFIX   => $suffix,
+                           DIR      => $dir,
+                           UNLINK   => 0,
+                         )
+    or $error ||= "can't open temporary file to store upload: $!\n";
+
+  unless ($error) {
+    while(<$fh>) {
+      print $sh $_;
+    }
+    $sh->filename =~ m!.*/([.\w]+)$!;
+    push @filenames,  "$field:$1";
+    close $sh
+  }
+
+}
+
+$error = "No files" unless scalar(@filenames);
+
+
diff --git a/httemplate/misc/process/cust_main-import.cgi b/httemplate/misc/process/cust_main-import.cgi
index 2568d1c7b..df61eb632 100644
--- a/httemplate/misc/process/cust_main-import.cgi
+++ b/httemplate/misc/process/cust_main-import.cgi
@@ -1,42 +1,9 @@
-% if ( $error ) {
-%   errorpage($error);
-%  } else {
-    <% include('/elements/header.html','Import successful') %> 
-    <% include('/elements/footer.html') %> 
-%  }
+<% $server->process %>
 <%init>
 
 die "access denied"
   unless $FS::CurrentUser::CurrentUser->access_right('Import');
 
-my $fh = $cgi->upload('file');
-my $error = '';
-if ( defined($fh) ) {
-
-  my $type;
-  if ( $cgi->param('file') =~ /\.(\w+)$/i ) {
-    $type = lc($1);
-  } else {
-    #or error out???
-    warn "can't parse file type from filename ". $cgi->param('file').
-         '; defaulting to CSV';
-    $type = 'csv';
-  }
-
-  $error =
-    FS::cust_main::batch_import( {
-      filehandle => $fh,
-      type       => $type,
-      agentnum   => scalar($cgi->param('agentnum')),
-      refnum     => scalar($cgi->param('refnum')),
-      pkgpart    => scalar($cgi->param('pkgpart')),
-      #'fields'    => [qw( cust_pkg.setup dayphone first last address1 address2
-      #                    city state zip comments                          )],
-      'format'   => scalar($cgi->param('format')),
-    } );
-
-} else {
-  $error = 'No file';
-}
+my $server = new FS::UI::Web::JSRPC 'FS::cust_main::process_batch_import', $cgi;
 
 
diff --git a/httemplate/search/cust_main.html b/httemplate/search/cust_main.html
index cd5e51f25..3282f0f31 100755
--- a/httemplate/search/cust_main.html
+++ b/httemplate/search/cust_main.html
@@ -43,7 +43,9 @@ my %search_hash = ();
 #$search_hash{'query'} = $cgi->keywords;
 
 #scalars
-for my $param (qw( agentnum status cancelled_pkgs cust_fields flattened_pkgs)) {
+for my $param (qw(
+  agentnum status cancelled_pkgs cust_fields flattened_pkgs custbatch
+)) {
   $search_hash{$param} = scalar( $cgi->param($param) )
     if $cgi->param($param);
 }