1 package FS::cust_main::Import;
4 use vars qw( $DEBUG $conf );
8 use File::Slurp qw( slurp );
9 use FS::Misc::DateTime qw( parse_datetime );
10 use FS::UID qw( dbh );
11 use FS::Record qw( qsearchs );
17 use FS::part_referral;
21 install_callback FS::UID sub {
27 FS::cust_main::Import - Batch customer importing
31 use FS::cust_main::Import;
34 FS::cust_main::Import::batch_import( {
35 file => $file, #filename
36 type => $type, #csv or xls
37 format => $format, #extended, extended-plus_company, svc_external,
38 #extended-plus_company_and_options
39 #extended-plus_options, or svc_external_svc_phone
40 agentnum => $agentnum,
43 job => $job, #optional job queue job, for progressbar updates
44 custbatch => $custbatch, #optional batch unique identifier
49 use FS::UI::Web::JSRPC;
51 new FS::UI::Web::JSRPC 'FS::cust_main::Import::process_batch_import', $cgi;
52 print $server->process;
56 Batch customer importing.
60 =item process_batch_import
62 Load a batch import as a queued JSRPC job
66 sub process_batch_import {
69 my $param = thaw(decode_base64(shift));
70 warn Dumper($param) if $DEBUG;
72 my $files = $param->{'uploaded_files'}
73 or die "No files provided.\n";
75 my (%files) = map { /^(\w+):([\.\w]+)$/ ? ($1,$2):() } split /,/, $files;
77 my $dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc. '/';
78 my $file = $dir. $files{'file'};
81 if ( $file =~ /\.(\w+)$/i ) {
85 warn "can't parse file type from filename $file; defaulting to CSV";
90 FS::cust_main::Import::batch_import( {
94 custbatch => $param->{custbatch},
95 agentnum => $param->{'agentnum'},
96 refnum => $param->{'refnum'},
97 pkgpart => $param->{'pkgpart'},
98 #'fields' => [qw( cust_pkg.setup dayphone first last address1 address2
99 # city state zip comments )],
100 'format' => $param->{'format'},
105 die "$error\n" if $error;
114 #some false laziness w/cdr.pm now
118 my $job = $param->{job};
120 my $filename = $param->{file};
121 my $type = $param->{type} || 'csv';
123 my $custbatch = $param->{custbatch};
125 my $agentnum = $param->{agentnum};
126 my $refnum = $param->{refnum};
127 my $pkgpart = $param->{pkgpart};
129 my $format = $param->{'format'};
133 if ( $format eq 'simple' ) {
134 @fields = qw( cust_pkg.setup dayphone first last
135 address1 address2 city state zip comments );
137 } elsif ( $format eq 'extended' ) {
138 @fields = qw( agent_custid refnum
139 last first address1 address2 city state zip country
141 ship_last ship_first ship_address1 ship_address2
142 ship_city ship_state ship_zip ship_country
143 payinfo paycvv paydate
146 svc_acct.username svc_acct._password
149 } elsif ( $format eq 'extended-plus_options' ) {
150 @fields = qw( agent_custid refnum
151 last first address1 address2 city state zip country
153 ship_last ship_first ship_company ship_address1 ship_address2
154 ship_city ship_state ship_zip ship_country
155 payinfo paycvv paydate
158 svc_acct.username svc_acct._password
162 } elsif ( $format eq 'extended-plus_company' ) {
163 @fields = qw( agent_custid refnum
164 last first company address1 address2 city state zip country
166 ship_last ship_first ship_company ship_address1 ship_address2
167 ship_city ship_state ship_zip ship_country
168 payinfo paycvv paydate
171 svc_acct.username svc_acct._password
174 } elsif ( $format eq 'extended-plus_company_and_options' ) {
175 @fields = qw( agent_custid refnum
176 last first company address1 address2 city state zip country
178 ship_last ship_first ship_company ship_address1 ship_address2
179 ship_city ship_state ship_zip ship_country
180 payinfo paycvv paydate
183 svc_acct.username svc_acct._password
187 } elsif ( $format =~ /^svc_external/ ) {
188 @fields = qw( agent_custid refnum
189 last first company address1 address2 city state zip country
191 ship_last ship_first ship_company ship_address1 ship_address2
192 ship_city ship_state ship_zip ship_country
193 payinfo paycvv paydate
195 cust_pkg.pkgpart cust_pkg.bill
196 svc_external.id svc_external.title
198 push @fields, map "svc_phone.$_", qw( countrycode phonenum sip_password pin)
199 if $format eq 'svc_external_svc_phone';
201 } elsif ( $format eq 'birthdates-acct_phone_hardware') {
202 @fields = qw( agent_custid refnum
203 last first company address1 address2 city state zip country
205 ship_last ship_first ship_company ship_address1 ship_address2
206 ship_city ship_state ship_zip ship_country
207 birthdate spouse_birthdate
208 payinfo paycvv paydate
210 cust_pkg.pkgpart cust_pkg.bill
211 svc_acct.username svc_acct._password
213 push @fields, map "svc_phone.$_", qw(countrycode phonenum sip_password pin);
214 push @fields, map "svc_hardware.$_", qw(typenum ip_addr hw_addr serial);
218 die "unknown format $format";
224 if ( $type eq 'csv' ) {
226 eval "use Text::CSV_XS;";
229 $parser = new Text::CSV_XS;
231 @buffer = split(/\r?\n/, slurp($filename) );
232 $count = scalar(@buffer);
234 } elsif ( $type eq 'xls' ) {
236 eval "use Spreadsheet::ParseExcel;";
239 my $excel = Spreadsheet::ParseExcel::Workbook->new->Parse($filename);
240 $parser = $excel->{Worksheet}[0]; #first sheet
242 $count = $parser->{MaxRow} || $parser->{MinRow};
246 die "Unknown file type $type\n";
251 local $SIG{HUP} = 'IGNORE';
252 local $SIG{INT} = 'IGNORE';
253 local $SIG{QUIT} = 'IGNORE';
254 local $SIG{TERM} = 'IGNORE';
255 local $SIG{TSTP} = 'IGNORE';
256 local $SIG{PIPE} = 'IGNORE';
258 my $oldAutoCommit = $FS::UID::AutoCommit;
259 local $FS::UID::AutoCommit = 0;
262 #implies ignore_expired_card
263 local($FS::cust_main::import) = 1;
264 local($FS::cust_main::import) = 1;
268 my( $last, $min_sec ) = ( time, 5 ); #progressbar foo
272 if ( $type eq 'csv' ) {
274 last unless scalar(@buffer);
275 $line = shift(@buffer);
277 $parser->parse($line) or do {
278 $dbh->rollback if $oldAutoCommit;
279 return "can't parse: ". $parser->error_input();
281 @columns = $parser->fields();
283 } elsif ( $type eq 'xls' ) {
285 last if $row > ($parser->{MaxRow} || $parser->{MinRow})
286 || ! $parser->{Cells}[$row];
288 my @row = @{ $parser->{Cells}[$row] };
289 @columns = map $_->{Val}, @row;
292 #warn $z++. ": $_\n" for @columns;
295 die "Unknown file type $type\n";
298 #warn join('-',@columns);
301 custbatch => $custbatch,
302 agentnum => $agentnum,
304 country => $conf->config('countrydefault') || 'US',
305 payby => $payby, #default
306 paydate => '12/2037', #default
309 my %cust_pkg = ( pkgpart => $pkgpart );
311 foreach my $field ( @fields ) {
313 if ( $field =~ /^cust_pkg\.(pkgpart|setup|bill|susp|adjourn|expire|cancel)$/ ) {
315 #$cust_pkg{$1} = parse_datetime( shift @$columns );
316 if ( $1 eq 'pkgpart' ) {
317 $cust_pkg{$1} = shift @columns;
318 } elsif ( $1 eq 'setup' ) {
319 $billtime = parse_datetime(shift @columns);
321 $cust_pkg{$1} = parse_datetime( shift @columns );
324 } elsif ( $field =~ /^svc_acct\.(username|_password)$/ ) {
326 $svc_x{$1} = shift @columns;
328 } elsif ( $field =~ /^svc_external\.(id|title)$/ ) {
330 $svc_x{$1} = shift @columns;
332 } elsif ( $field =~ /^svc_phone\.(countrycode|phonenum|sip_password|pin)$/ ) {
333 $svc_x{$1} = shift @columns;
335 } elsif ( $field =~ /^svc_hardware\.(typenum|ip_addr|hw_addr|serial)$/ ) {
337 $svc_x{$1} = shift @columns;
342 if ( $field eq 'refnum' && $columns[0] !~ /^\s*(\d+)\s*$/ ) {
344 my $referral = $columns[0];
345 my %hash = ( 'referral' => $referral,
346 'agentnum' => $agentnum,
350 my $part_referral = qsearchs('part_referral', \%hash )
351 || new FS::part_referral \%hash;
353 unless ( $part_referral->refnum ) {
354 my $error = $part_referral->insert;
356 $dbh->rollback if $oldAutoCommit;
357 return "can't auto-insert advertising source: $referral: $error";
361 $columns[0] = $part_referral->refnum;
364 my $value = shift @columns;
365 $cust_main{$field} = $value if length($value);
369 if ( defined $cust_main{'payinfo'} && length $cust_main{'payinfo'} ) {
370 $cust_main{'payby'} = 'CARD';
371 if ($cust_main{'payinfo'} =~ /\s*([AD]?)(.*)\s*$/) {
372 $cust_main{'payby'} = 'DCRD' if $1 eq 'D';
373 $cust_main{'payinfo'} = $2;
377 $cust_main{$_} = parse_datetime($cust_main{$_})
378 foreach grep $cust_main{$_}, qw( birthdate spouse_birthdate );
380 my $invoicing_list = $cust_main{'invoicing_list'}
381 ? [ delete $cust_main{'invoicing_list'} ]
384 my $customer_options = delete $cust_main{customer_options};
385 $cust_main{tax} = 'Y' if $customer_options =~ /taxexempt/i;
386 push @$invoicing_list, 'POST' if $customer_options =~ /postalinvoice/i;
388 my $cust_main = new FS::cust_main ( \%cust_main );
391 tie my %hash, 'Tie::RefHash'; #this part is important
393 if ( $cust_pkg{'pkgpart'} ) {
395 unless ( $cust_pkg{'pkgpart'} =~ /^\d+$/ ) {
396 $dbh->rollback if $oldAutoCommit;
397 return 'illegal pkgpart: '. $cust_pkg{'pkgpart'};
400 my $cust_pkg = new FS::cust_pkg ( \%cust_pkg );
404 if ( $svc_x{'username'} ) {
406 } elsif ( $svc_x{'id'} || $svc_x{'title'} ) {
407 $svcdb = 'svc_external';
411 if ( $svc_x{'countrycode'} || $svc_x{'phonenum'} ) {
412 $svc_phone = FS::svc_phone->new( {
413 map { $_ => delete($svc_x{$_}) }
414 qw( countrycode phonenum sip_password pin )
418 my $svc_hardware = '';
419 if ( $svc_x{'typenum'} ) {
420 $svc_hardware = FS::svc_hardware->new( {
421 map { $_ => delete($svc_x{$_}) }
422 qw( typenum ip_addr hw_addr serial )
426 if ( $svcdb || $svc_phone || $svc_hardware ) {
427 my $part_pkg = $cust_pkg->part_pkg;
428 unless ( $part_pkg ) {
429 $dbh->rollback if $oldAutoCommit;
430 return "unknown pkgpart: ". $cust_pkg{'pkgpart'};
433 $svc_x{svcpart} = $part_pkg->svcpart_unique_svcdb( $svcdb );
434 my $class = "FS::$svcdb";
435 push @svc_x, $class->new( \%svc_x );
438 $svc_phone->svcpart( $part_pkg->svcpart_unique_svcdb('svc_phone') );
439 push @svc_x, $svc_phone;
441 if ( $svc_hardware ) {
442 $svc_hardware->svcpart( $part_pkg->svcpart_unique_svcdb('svc_hardware') );
443 push @svc_x, $svc_hardware;
448 $hash{$cust_pkg} = \@svc_x;
451 my $error = $cust_main->insert( \%hash, $invoicing_list );
454 $dbh->rollback if $oldAutoCommit;
455 return "can't insert customer". ( $line ? " for $line" : '' ). ": $error";
458 if ( $format eq 'simple' ) {
460 #false laziness w/bill.cgi
461 $error = $cust_main->bill( 'time' => $billtime );
463 $dbh->rollback if $oldAutoCommit;
464 return "can't bill customer for $line: $error";
467 $error = $cust_main->apply_payments_and_credits;
469 $dbh->rollback if $oldAutoCommit;
470 return "can't bill customer for $line: $error";
473 $error = $cust_main->collect();
475 $dbh->rollback if $oldAutoCommit;
476 return "can't collect customer for $line: $error";
483 if ( $job && time - $min_sec > $last ) { #progress bar
484 $job->update_statustext( int(100 * $row / $count) );
490 $dbh->commit or die $dbh->errstr if $oldAutoCommit;;
492 return "Empty file!" unless $row;
500 Not enough documentation.
504 L<FS::cust_main>, L<FS::cust_pkg>,
505 L<FS::svc_acct>, L<FS::svc_external>, L<FS::svc_phone>