1 package FS::cust_main::Import;
4 use vars qw( $DEBUG $conf );
6 use File::Slurp qw( slurp );
7 use FS::Misc::DateTime qw( parse_datetime );
9 use FS::Record qw( qsearchs );
12 use FS::svc_broadband;
16 use FS::part_referral;
17 use Business::CreditCard 0.35;
21 install_callback FS::UID sub {
25 my %is_location = map { $_ => 1 } FS::cust_main::Location->location_fields;
29 FS::cust_main::Import - Batch customer importing
33 use FS::cust_main::Import;
36 FS::cust_main::Import::batch_import( {
37 file => $file, #filename
38 type => $type, #csv or xls
39 format => $format, #extended, extended-plus_company, svc_external,
40 #extended-plus_company_and_options
41 #extended-plus_options, or svc_external_svc_phone
42 agentnum => $agentnum,
45 job => $job, #optional job queue job, for progressbar updates
46 custbatch => $custbatch, #optional batch unique identifier
51 use FS::UI::Web::JSRPC;
53 new FS::UI::Web::JSRPC 'FS::cust_main::Import::process_batch_import', $cgi;
54 print $server->process;
58 Batch customer importing.
62 =item process_batch_import
64 Load a batch import as a queued JSRPC job
68 sub process_batch_import {
71 warn Dumper($param) if $DEBUG;
73 my $files = $param->{'uploaded_files'}
74 or die "No files provided.\n";
76 my (%files) = map { /^(\w+):([\.\w]+)$/ ? ($1,$2):() } split /,/, $files;
78 my $dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc. '/';
79 my $file = $dir. $files{'file'};
82 if ( $file =~ /\.(\w+)$/i ) {
86 warn "can't parse file type from filename $file; defaulting to CSV";
91 FS::cust_main::Import::batch_import( {
95 custbatch => $param->{custbatch},
96 agentnum => $param->{'agentnum'},
97 refnum => $param->{'refnum'},
98 pkgpart => $param->{'pkgpart'},
99 validate_cc => $param->{'validate_cc'},
100 #'fields' => [qw( cust_pkg.setup dayphone first last address1 address2
101 # city state zip comments )],
102 'format' => $param->{'format'},
107 die "$error\n" if $error;
116 #some false laziness w/cdr.pm now
120 my $job = $param->{job};
122 my $filename = $param->{file};
123 my $type = $param->{type} || 'csv';
125 my $custbatch = $param->{custbatch};
127 my $agentnum = $param->{agentnum};
128 my $refnum = $param->{refnum};
129 my $pkgpart = $param->{pkgpart};
130 my $validate_cc = $param->{validate_cc};
132 my $format = $param->{'format'};
135 if ( $format eq 'simple' ) {
136 @fields = qw( cust_pkg.setup dayphone first last
137 address1 address2 city state zip comments );
138 } elsif ( $format eq 'extended' ) {
139 @fields = qw( agent_custid refnum
140 last first address1 address2 city state zip country
142 ship_last ship_first ship_address1 ship_address2
143 ship_city ship_state ship_zip ship_country
144 payinfo paycvv paydate
147 svc_acct.username svc_acct._password
149 } elsif ( $format eq 'extended-plus_options' ) {
150 @fields = qw( agent_custid refnum
151 last first address1 address2 city state zip country
153 ship_last ship_first ship_address1 ship_address2
154 ship_city ship_state ship_zip ship_country
155 payinfo paycvv paydate
158 svc_acct.username svc_acct._password
161 } elsif ( $format eq 'extended-plus_company' ) {
162 @fields = qw( agent_custid refnum
163 last first company address1 address2 city state zip country
165 ship_last ship_first ship_company ship_address1 ship_address2
166 ship_city ship_state ship_zip ship_country
167 payinfo paycvv paydate
170 svc_acct.username svc_acct._password
172 } elsif ( $format eq 'extended-plus_company_and_options' ) {
173 @fields = qw( agent_custid refnum
174 last first company address1 address2 city state zip country
176 ship_last ship_first ship_company ship_address1 ship_address2
177 ship_city ship_state ship_zip ship_country
178 payinfo paycvv paydate
181 svc_acct.username svc_acct._password
184 } elsif ( $format =~ /^svc_broadband/ ) {
185 @fields = qw( agent_custid refnum
186 last first company address1 address2 city state zip country
188 ship_last ship_first ship_company ship_address1 ship_address2
189 ship_city ship_state ship_zip ship_country
190 payinfo paycvv paydate
192 cust_pkg.pkgpart cust_pkg.bill
194 push @fields, map "svc_broadband.$_", qw( ip_addr mac_addr sectornum );
195 } elsif ( $format =~ /^svc_external/ ) {
196 @fields = qw( agent_custid refnum
197 last first company address1 address2 city state zip country
199 ship_last ship_first ship_company ship_address1 ship_address2
200 ship_city ship_state ship_zip ship_country
201 payinfo paycvv paydate
203 cust_pkg.pkgpart cust_pkg.bill
204 svc_external.id svc_external.title
206 push @fields, map "svc_phone.$_", qw( countrycode phonenum sip_password pin)
207 if $format eq 'svc_external_svc_phone';
208 } elsif ( $format eq 'birthdates-acct_phone_hardware') {
209 @fields = qw( agent_custid refnum
210 last first company address1 address2 city state zip country
212 ship_last ship_first ship_company ship_address1 ship_address2
213 ship_city ship_state ship_zip ship_country
214 birthdate spouse_birthdate
215 payinfo paycvv paydate
217 cust_pkg.pkgpart cust_pkg.bill
218 svc_acct.username svc_acct._password
220 push @fields, map "svc_phone.$_", qw(countrycode phonenum sip_password pin);
221 push @fields, map "svc_hardware.$_", qw(typenum ip_addr hw_addr serial);
223 } elsif ( $format eq 'national_id-acct_phone') {
224 @fields = qw( agent_custid refnum
225 last first company address1 address2 city state zip country
227 ship_last ship_first ship_company ship_address1 ship_address2
228 ship_city ship_state ship_zip ship_country
230 payinfo paycvv paydate
232 cust_pkg.pkgpart cust_pkg.bill
233 svc_acct.username svc_acct._password svc_acct.slipip
235 push @fields, map "svc_phone.$_", qw(countrycode phonenum sip_password pin);
238 die "unknown format $format";
244 if ( $type eq 'csv' ) {
246 eval "use Text::CSV_XS;";
249 $parser = new Text::CSV_XS;
251 @buffer = split(/\r?\n/, slurp($filename) );
252 $count = scalar(@buffer);
254 } elsif ( $type eq 'xls' ) {
256 eval "use Spreadsheet::ParseExcel;";
259 my $excel = Spreadsheet::ParseExcel::Workbook->new->Parse($filename);
260 $parser = $excel->{Worksheet}[0]; #first sheet
262 $count = $parser->{MaxRow} || $parser->{MinRow};
266 die "Unknown file type $type\n";
271 local $SIG{HUP} = 'IGNORE';
272 local $SIG{INT} = 'IGNORE';
273 local $SIG{QUIT} = 'IGNORE';
274 local $SIG{TERM} = 'IGNORE';
275 local $SIG{TSTP} = 'IGNORE';
276 local $SIG{PIPE} = 'IGNORE';
278 my $oldAutoCommit = $FS::UID::AutoCommit;
279 local $FS::UID::AutoCommit = 0;
282 #implies ignore_expired_card
283 local($FS::cust_main::import) = 1;
284 local($FS::cust_main::import) = 1;
288 my( $last, $min_sec ) = ( time, 5 ); #progressbar foo
292 if ( $type eq 'csv' ) {
294 last unless scalar(@buffer);
295 $line = shift(@buffer);
297 $parser->parse($line) or do {
298 $dbh->rollback if $oldAutoCommit;
299 return "can't parse: ". $parser->error_input();
301 @columns = $parser->fields();
303 } elsif ( $type eq 'xls' ) {
305 last if $row > ($parser->{MaxRow} || $parser->{MinRow})
306 || ! $parser->{Cells}[$row];
308 my @row = @{ $parser->{Cells}[$row] };
309 @columns = map $_->{Val}, @row;
312 #warn $z++. ": $_\n" for @columns;
315 die "Unknown file type $type\n";
318 #warn join('-',@columns);
321 custbatch => $custbatch,
322 agentnum => $agentnum,
324 paydate => '12/2037', #default
327 my %cust_pkg = ( pkgpart => $pkgpart );
329 my %bill_location = ();
330 my %ship_location = ();
332 foreach my $field ( @fields ) {
334 if ( $field =~ /^cust_pkg\.(pkgpart|setup|bill|susp|adjourn|expire|cancel)$/ ) {
336 #$cust_pkg{$1} = parse_datetime( shift @$columns );
337 if ( $1 eq 'pkgpart' ) {
338 $cust_pkg{$1} = shift @columns;
339 } elsif ( $1 eq 'setup' ) {
340 $billtime = parse_datetime(shift @columns);
342 $cust_pkg{$1} = parse_datetime( shift @columns );
345 } elsif ( $field =~ /^svc_acct\.(username|_password|slipip)$/ ) {
347 $svc_x{$1} = shift @columns;
349 } elsif ( $field =~ /^svc_broadband\.(ip_addr|mac_addr|sectornum)$/ ) {
351 $svc_x{$1} = shift @columns;
353 } elsif ( $field =~ /^svc_external\.(id|title)$/ ) {
355 $svc_x{$1} = shift @columns;
357 } elsif ( $field =~ /^svc_phone\.(countrycode|phonenum|sip_password|pin)$/ ) {
358 $svc_x{$1} = shift @columns;
360 } elsif ( $field =~ /^svc_hardware\.(typenum|ip_addr|hw_addr|serial)$/ ) {
362 $svc_x{$1} = shift @columns;
364 } elsif ( $is_location{$field} ) {
366 $bill_location{$field} = shift @columns;
368 } elsif ( $field =~ /^ship_(.*)$/ and $is_location{$1} ) {
370 $ship_location{$1} = shift @columns;
375 if ( $field eq 'refnum' && $columns[0] !~ /^\s*(\d+)\s*$/ ) {
377 my $referral = $columns[0];
378 my %hash = ( 'referral' => $referral,
379 'agentnum' => $agentnum,
383 my $part_referral = qsearchs('part_referral', \%hash )
384 || new FS::part_referral \%hash;
386 unless ( $part_referral->refnum ) {
387 my $error = $part_referral->insert;
389 $dbh->rollback if $oldAutoCommit;
390 return "can't auto-insert advertising source: $referral: $error";
394 $columns[0] = $part_referral->refnum;
397 my $value = shift @columns;
398 $cust_main{$field} = $value if length($value);
400 } # foreach my $field
401 # finished importing columns
403 $bill_location{'country'} ||= $conf->config('countrydefault') || 'US';
404 $cust_main{'bill_location'} = FS::cust_location->new(\%bill_location);
405 if ( grep $_, values(%ship_location) ) {
406 $ship_location{'country'} ||= $conf->config('countrydefault') || 'US';
407 $cust_main{'ship_location'} = FS::cust_location->new(\%ship_location);
409 $cust_main{'ship_location'} = $cust_main{'bill_location'};
412 if ( defined $cust_main{'payinfo'} && length $cust_main{'payinfo'} ) {
414 if ( $cust_main{'payinfo'} =~ /^\s*(\d+\@[\d\.]+)\s*$/ ) {
416 delete $cust_main{'payinfo'};
418 $cust_payby = new FS::cust_payby {
423 } elsif ($cust_main{'payinfo'} =~ /^\s*([AD]?)(.*)\s*$/) {
425 delete $cust_main{'payinfo'};
427 ## validate credit card if requested
430 or return "Invalid card($2) for customer ".$cust_main{'first'}." ".$cust_main{'last'};
431 return "Unknown card type for customer ".$cust_main{'first'}." ".$cust_main{'last'}
432 if cardtype($2) eq "Unknown";
435 $cust_payby = new FS::cust_payby {
436 'payby' => ($1 eq 'D') ? 'DCRD' : 'CARD',
438 'paycvv' => delete $cust_main{'paycvv'},
439 'paydate' => delete $cust_main{'paydate'},
440 'payname' => $cust_main{'first'}. ' '. $cust_main{'last'},
447 $cust_main{$_} = parse_datetime($cust_main{$_})
448 foreach grep $cust_main{$_},
449 qw( birthdate spouse_birthdate anniversary_date );
451 my $invoicing_list = $cust_main{'invoicing_list'}
452 ? [ delete $cust_main{'invoicing_list'} ]
455 my $customer_options = delete $cust_main{customer_options};
456 $cust_main{tax} = 'Y' if $customer_options =~ /taxexempt/i;
457 push @$invoicing_list, 'POST' if $customer_options =~ /postalinvoice/i;
459 my $cust_main = new FS::cust_main ( \%cust_main );
462 tie my %hash, 'Tie::RefHash'; #this part is important
464 if ( $cust_pkg{'pkgpart'} ) {
466 unless ( $cust_pkg{'pkgpart'} =~ /^\d+$/ ) {
467 $dbh->rollback if $oldAutoCommit;
468 return 'illegal pkgpart: '. $cust_pkg{'pkgpart'};
471 my $cust_pkg = new FS::cust_pkg ( \%cust_pkg );
475 if ( $svc_x{'username'} ) {
477 } elsif ( $svc_x{'id'} || $svc_x{'title'} ) {
478 $svcdb = 'svc_external';
479 } elsif ( $svc_x{ip_addr} || $svc_x{mac_addr} ) {
480 $svcdb = 'svc_broadband';
484 if ( $svc_x{'countrycode'} || $svc_x{'phonenum'} ) {
485 $svc_phone = FS::svc_phone->new( {
486 map { $_ => delete($svc_x{$_}) }
487 qw( countrycode phonenum sip_password pin )
491 my $svc_hardware = '';
492 if ( $svc_x{'typenum'} ) {
493 $svc_hardware = FS::svc_hardware->new( {
494 map { $_ => delete($svc_x{$_}) }
495 qw( typenum ip_addr hw_addr serial )
499 if ( $svcdb || $svc_phone || $svc_hardware ) {
500 my $part_pkg = $cust_pkg->part_pkg;
501 unless ( $part_pkg ) {
502 $dbh->rollback if $oldAutoCommit;
503 return "unknown pkgpart: ". $cust_pkg{'pkgpart'};
506 $svc_x{svcpart} = $part_pkg->svcpart_unique_svcdb( $svcdb );
507 my $class = "FS::$svcdb";
508 push @svc_x, $class->new( \%svc_x );
511 $svc_phone->svcpart( $part_pkg->svcpart_unique_svcdb('svc_phone') );
512 push @svc_x, $svc_phone;
514 if ( $svc_hardware ) {
515 $svc_hardware->svcpart( $part_pkg->svcpart_unique_svcdb('svc_hardware') );
516 push @svc_x, $svc_hardware;
521 $hash{$cust_pkg} = \@svc_x;
524 my %options = ('invoicing_list' => $invoicing_list);
525 $options{'cust_payby'} = [ $cust_payby ] if $cust_payby;
527 my $error = $cust_main->insert( \%hash, %options );
530 $dbh->rollback if $oldAutoCommit;
531 return "can't insert customer". ( $line ? " for $line" : '' ). ": $error";
534 if ( $format eq 'simple' ) {
536 #false laziness w/bill.cgi
537 $error = $cust_main->bill( 'time' => $billtime );
539 $dbh->rollback if $oldAutoCommit;
540 return "can't bill customer for $line: $error";
543 $error = $cust_main->apply_payments_and_credits;
545 $dbh->rollback if $oldAutoCommit;
546 return "can't bill customer for $line: $error";
549 $error = $cust_main->collect();
551 $dbh->rollback if $oldAutoCommit;
552 return "can't collect customer for $line: $error";
559 if ( $job && time - $min_sec > $last ) { #progress bar
560 $job->update_statustext( int(100 * $row / $count) );
566 $dbh->commit or die $dbh->errstr if $oldAutoCommit;;
568 return "Empty file!" unless $row;
576 Not enough documentation.
580 L<FS::cust_main>, L<FS::cust_pkg>,
581 L<FS::svc_acct>, L<FS::svc_external>, L<FS::svc_phone>