use Scalar::Util qw( blessed );
use File::CounterFile;
use Locale::Country;
+use Text::CSV_XS;
+use File::Slurp qw( slurp );
use DBI qw(:sql_types);
use DBIx::DBSchema 0.33;
use FS::UID qw(dbh getotaker datasrc driver_name);
if ( $type =~ /(int|(big)?serial)/i && $value =~ /^\d+(\.\d+)?$/ ) {
$TYPE = SQL_INTEGER;
- #DBD::Pg 1.49: Cannot bind ... unknown sql_type 6
- #} elsif ( ( $type =~ /(numeric)/i && $value =~ /^[+-]?\d+(\.\d+)?$/)
- # || ( $type =~ /(real|float4)/i
- # && $value =~ /[-+]?\d*\.?\d+([eE][-+]?\d+)?/
- # )
- # ) {
- # $TYPE = SQL_FLOAT;
+ #DBD::Pg 1.49: Cannot bind ... unknown sql_type 6 with SQL_FLOAT
+ } elsif ( ( $type =~ /(numeric)/i && $value =~ /^[+-]?\d+(\.\d+)?$/)
+ || ( $type =~ /(real|float4)/i
+ && $value =~ /[-+]?\d*\.?\d+([eE][-+]?\d+)?/
+ )
+ ) {
+ $TYPE = SQL_DECIMAL;
}
if ( $DEBUG > 2 ) {
} );
}
+=item lock_table
+
+Locks this table with a database-driver specific lock method. This is used
+as a mutex in order to do a duplicate search.
+
+For PostgreSQL, does "LOCK TABLE tablename IN SHARE ROW EXCLUSIVE MODE".
+
+For MySQL, does a SELECT FOR UPDATE on the duplicate_lock table.
+
+Errors are fatal; no useful return value.
+
+Note: To use this method for new tables other than svc_acct and svc_phone,
+edit freeside-upgrade and add those tables to the duplicate_lock list.
+
+=cut
+
+sub lock_table {
+ my $self = shift;
+ my $table = $self->table;
+
+ warn "$me locking $table table\n" if $DEBUG;
+
+ if ( driver_name =~ /^Pg/i ) {
+
+ dbh->do("LOCK TABLE $table IN SHARE ROW EXCLUSIVE MODE")
+ or die dbh->errstr;
+
+ } elsif ( driver_name =~ /^mysql/i ) {
+
+ dbh->do("SELECT * FROM duplicate_lock
+ WHERE lockname = '$table'
+ FOR UPDATE"
+ ) or die dbh->errstr;
+
+ } else {
+
+ die "unknown database ". driver_name. "; don't know how to lock table";
+
+ }
+
+ warn "$me acquired $table table lock\n" if $DEBUG;
+
+}
+
=item insert
Inserts this record to the database. If there is an error, returns the error,
'';
}
+=item process_batch_import JOB OPTIONS_HASHREF PARAMS
+
+Processes a batch import as a queued JSRPC job
+
+JOB is an FS::queue entry.
+
+OPTIONS_HASHREF can have the following keys:
+
+=over 4
+
+=item table
+
+Table name (required).
+
+=item params
+
+Listref of field names for static fields. They will be given values from the
+PARAMS hashref and passed as a "params" hashref to batch_import.
+
+=item formats
+
+Formats hashref. Keys are field names, values are listrefs that define the
+format.
+
+Each listref value can be a column name or a code reference. Coderefs are run
+with the row object and data as the two parameters. For example, this coderef
+does the same thing as using the "columnname" string:
+
+ sub {
+ my( $record, $data ) = @_;
+ $record->columnname( $data );
+ },
+
+=item format_types
+
+Optional format hashref of types. Keys are field names, values are "csv",
+"xls" or "fixedlength". Overrides automatic determination of file type
+from extension.
+
+=item format_headers
+
+Optional format hashref of header lines. Keys are field names, values are 0
+for no header, 1 to ignore the first line, or to higher numbers to ignore that
+number of lines.
+
+=item format_sep_chars
+
+Optional format hashref of CSV sep_chars. Keys are field names, values are the
+CSV separation character.
+
+=item format_fixedlenth_formats
+
+Optional format hashref of fixed length format defintiions. Keys are field
+names, values Parse::FixedLength listrefs of field definitions.
+
+=item default_csv
+
+Set true to default to CSV file type if the filename does not contain a
+recognizable ".csv" or ".xls" extension (and type is not pre-specified by
+format_types).
+
+=back
+
+PARAMS is a base64-encoded Storable string containing the POSTed data as
+a hash ref. It normally contains at least one field, "uploaded files",
+generated by /elements/file-upload.html and containing the list of uploaded
+files. Currently only supports a single file named "file".
+
+=cut
+
+use Storable qw(thaw);
+use Data::Dumper;
+use MIME::Base64;
+sub process_batch_import {
+ my($job, $opt) = ( shift, shift );
+
+ my $table = $opt->{table};
+ my @pass_params = @{ $opt->{params} };
+ my %formats = %{ $opt->{formats} };
+
+ my $param = thaw(decode_base64(shift));
+ warn Dumper($param) if $DEBUG;
+
+ my $files = $param->{'uploaded_files'}
+ or die "No files provided.\n";
+
+ my (%files) = map { /^(\w+):([\.\w]+)$/ ? ($1,$2):() } split /,/, $files;
+
+ my $dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc. '/';
+ my $file = $dir. $files{'file'};
+
+ my $type = $opt->{'format_types'}
+ ? $opt->{'format_types'}{ $param->{'format'} }
+ : '';
+
+ unless ( $type ) {
+ if ( $file =~ /\.(\w+)$/i ) {
+ $type = lc($1);
+ } else {
+ #or error out???
+ warn "can't parse file type from filename $file; defaulting to CSV";
+ $type = 'csv';
+ }
+ $type = 'csv'
+ if $opt->{'default_csv'} && $type ne 'xls';
+ }
+
+ my $header = $opt->{'format_headers'}
+ ? $opt->{'format_headers'}{ $param->{'format'} }
+ : 0;
+
+ my $sep_char = $opt->{'format_sep_chars'}
+ ? $opt->{'format_sep_chars'}{ $param->{'format'} }
+ : ',';
+
+ my $fixedlength_format =
+ $opt->{'format_fixedlength_formats'}
+ ? $opt->{'format_fixedlength_formats'}{ $param->{'format'} }
+ : '';
+
+ my $error =
+ FS::Record::batch_import( {
+ table => $table,
+ formats => \%formats,
+ job => $job,
+ file => $file,
+ type => $type,
+ format => $param->{format},
+ header => $header,
+ sep_char => $sep_char,
+ fixedlength_format => $fixedlength_format,
+ params => { map { $_ => $param->{$_} } @pass_params },
+ } );
+
+ unlink $file;
+
+ die "$error\n" if $error;
+}
+
+=item batch_import PARAM_HASHREF
+
+Class method for batch imports. Available params:
+
+=over 4
+
+=item table
+
+=item formats
+
+=item params
+
+=item job
+
+FS::queue object, will be updated with progress
+
+=item filename
+
+=item type
+
+csv, xls or fixedlength
+
+=item format
+
+=item header
+
+=item sep_char
+
+=item fixedlength_format
+
+=item empty_ok
+
+=back
+
+=cut
+
+sub batch_import {
+ my $param = shift;
+
+ warn "$me batch_import call with params: \n". Dumper($param)
+ if $DEBUG;
+
+ my $table = $param->{table};
+ my $formats = $param->{formats};
+ my $params = $param->{params};
+
+ my $job = $param->{job};
+
+ my $filename = $param->{file};
+ my $type = $param->{type} || 'csv';
+
+ my $format = $param->{'format'};
+
+ die "unknown format $format" unless exists $formats->{ $format };
+ my @fields = @{ $formats->{ $format } };
+
+ my $row = 0;
+ my $count;
+ my $parser;
+ my @buffer = ();
+ if ( $type eq 'csv' || $type eq 'fixedlength' ) {
+
+ if ( $type eq 'csv' ) {
+
+ my %attr = ();
+ foreach ( grep exists($param->{$_}), qw( sep_char ) ) {
+ $attr{$_} = $param->{$_};
+ }
+
+ $parser = new Text::CSV_XS \%attr;
+
+ } elsif ( $type eq 'fixedlength' ) {
+
+ eval "use Parse::FixedLength;";
+ die $@ if $@;
+ $parser = new Parse::FixedLength $param->{'fixedlength_format'};
+
+ } else {
+ die "Unknown file type $type\n";
+ }
+
+ @buffer = split(/\r?\n/, slurp($filename) );
+ splice(@buffer, 0, ($param->{'header'} || 0) );
+ $count = scalar(@buffer);
+
+ } elsif ( $type eq 'xls' ) {
+
+ eval "use Spreadsheet::ParseExcel;";
+ die $@ if $@;
+
+ my $excel = Spreadsheet::ParseExcel::Workbook->new->Parse($filename);
+
+ $parser = $excel->{Worksheet}[0]; #first sheet
+
+ $count = $parser->{MaxRow} || $parser->{MinRow};
+ $count++;
+
+ $row = $param->{'header'} || 0;
+
+ } else {
+ die "Unknown file type $type\n";
+ }
+
+ #my $columns;
+
+ local $SIG{HUP} = 'IGNORE';
+ local $SIG{INT} = 'IGNORE';
+ local $SIG{QUIT} = 'IGNORE';
+ local $SIG{TERM} = 'IGNORE';
+ local $SIG{TSTP} = 'IGNORE';
+ local $SIG{PIPE} = 'IGNORE';
+
+ my $oldAutoCommit = $FS::UID::AutoCommit;
+ local $FS::UID::AutoCommit = 0;
+ my $dbh = dbh;
+
+ my $line;
+ my $imported = 0;
+ my( $last, $min_sec ) = ( time, 5 ); #progressbar foo
+ while (1) {
+
+ my @columns = ();
+ if ( $type eq 'csv' ) {
+
+ last unless scalar(@buffer);
+ $line = shift(@buffer);
+
+ $parser->parse($line) or do {
+ $dbh->rollback if $oldAutoCommit;
+ return "can't parse: ". $parser->error_input();
+ };
+ @columns = $parser->fields();
+
+ } elsif ( $type eq 'fixedlength' ) {
+
+ @columns = $parser->parse($line);
+
+ } elsif ( $type eq 'xls' ) {
+
+ last if $row > ($parser->{MaxRow} || $parser->{MinRow})
+ || ! $parser->{Cells}[$row];
+
+ my @row = @{ $parser->{Cells}[$row] };
+ @columns = map $_->{Val}, @row;
+
+ #my $z = 'A';
+ #warn $z++. ": $_\n" for @columns;
+
+ } else {
+ die "Unknown file type $type\n";
+ }
+
+ my @later = ();
+ my %hash = %$params;
+
+ foreach my $field ( @fields ) {
+
+ my $value = shift @columns;
+
+ if ( ref($field) eq 'CODE' ) {
+ #&{$field}(\%hash, $value);
+ push @later, $field, $value;
+ } else {
+ $hash{$field} = $value if length($value);
+ }
+
+ }
+
+ my $class = "FS::$table";
+
+ my $record = $class->new( \%hash );
+
+ while ( scalar(@later) ) {
+ my $sub = shift @later;
+ my $data = shift @later;
+ &{$sub}($record, $data); # $record->&{$sub}($data);
+ }
+
+ my $error = $record->insert;
+
+ if ( $error ) {
+ $dbh->rollback if $oldAutoCommit;
+ return "can't insert record". ( $line ? " for $line" : '' ). ": $error";
+ }
+
+ $imported++;
+
+ if ( $job && time - $min_sec > $last ) { #progress bar
+ $job->update_statustext( int(100 * $imported / $count) );
+ $last = time;
+ }
+
+ }
+
+ $dbh->commit or die $dbh->errstr if $oldAutoCommit;;
+
+ return "Empty file!" unless $imported || $param->{empty_ok};
+
+ ''; #no error
+
+}
+
sub _h_statement {
my( $self, $action, $time ) = @_;
$self->setfield($field,'');
} elsif ( $country eq 'US' || $country eq 'CA' ) {
$phonen =~ s/\D//g;
+ $phonen = $conf->config('cust_main-default_areacode').$phonen
+ if length($phonen)==7 && $conf->config('cust_main-default_areacode');
$phonen =~ /^(\d{3})(\d{3})(\d{4})(\d*)$/
or return gettext('illegal_phone'). " $field: ". $self->getfield($field);
$phonen = "$1-$2-$3";
projects / freeside.git / blobdiff