X-Git-Url: http://git.freeside.biz/gitweb/?a=blobdiff_plain;f=FS%2FFS%2FRecord.pm;h=16949fa95a49ff461b07392fa6ba21392343c677;hb=0da7ecb92ccd671c47cb7fd4d04560cc80d4b520;hp=1f0b140e22282a06c6bd94a38fb0eb876c4795c6;hpb=a3342227e5f64a2ef4d45ccd5c6101b325549f01;p=freeside.git diff --git a/FS/FS/Record.pm b/FS/FS/Record.pm index 1f0b140e2..16949fa95 100644 --- a/FS/FS/Record.pm +++ b/FS/FS/Record.pm @@ -766,6 +766,50 @@ sub select_for_update { } ); } +=item lock_table + +Locks this table with a database-driver specific lock method. This is used +as a mutex in order to do a duplicate search. + +For PostgreSQL, does "LOCK TABLE tablename IN SHARE ROW EXCLUSIVE MODE". + +For MySQL, does a SELECT FOR UPDATE on the duplicate_lock table. + +Errors are fatal; no useful return value. + +Note: To use this method for new tables other than svc_acct and svc_phone, +edit freeside-upgrade and add those tables to the duplicate_lock list. + +=cut + +sub lock_table { + my $self = shift; + my $table = $self->table; + + warn "$me locking $table table\n" if $DEBUG; + + if ( driver_name =~ /^Pg/i ) { + + dbh->do("LOCK TABLE $table IN SHARE ROW EXCLUSIVE MODE") + or die dbh->errstr; + + } elsif ( driver_name =~ /^mysql/i ) { + + dbh->do("SELECT * FROM duplicate_lock + WHERE lockname = '$table' + FOR UPDATE" + ) or die dbh->errstr; + + } else { + + die "unknown database ". driver_name. "; don't know how to lock table"; + + } + + warn "$me acquired $table table lock\n" if $DEBUG; + +} + =item insert Inserts this record to the database. If there is an error, returns the error, @@ -1296,18 +1340,74 @@ sub check { ''; } -=item batch_import PARAM_HASHREF +=item process_batch_import JOB OPTIONS_HASHREF PARAMS -Class method for batch imports. Available params: +Processes a batch import as a queued JSRPC job + +JOB is an FS::queue entry. + +OPTIONS_HASHREF can have the following keys: =over 4 -=item job +=item table -FS::queue object, will be updated with progress +Table name (required). + +=item params + +Listref of field names for static fields. They will be given values from the +PARAMS hashref and passed as a "params" hashref to batch_import. + +=item formats + +Formats hashref. Keys are field names, values are listrefs that define the +format. + +Each listref value can be a column name or a code reference. Coderefs are run +with the row object and data as the two parameters. For example, this coderef +does the same thing as using the "columnname" string: + + sub { + my( $record, $data ) = @_; + $record->columnname( $data ); + }, + +=item format_types + +Optional format hashref of types. Keys are field names, values are "csv", +"xls" or "fixedlength". Overrides automatic determination of file type +from extension. + +=item format_headers + +Optional format hashref of header lines. Keys are field names, values are 0 +for no header, 1 to ignore the first line, or to higher numbers to ignore that +number of lines. + +=item format_sep_chars + +Optional format hashref of CSV sep_chars. Keys are field names, values are the +CSV separation character. + +=item format_fixedlenth_formats + +Optional format hashref of fixed length format defintiions. Keys are field +names, values Parse::FixedLength listrefs of field definitions. + +=item default_csv + +Set true to default to CSV file type if the filename does not contain a +recognizable ".csv" or ".xls" extension (and type is not pre-specified by +format_types). =back +PARAMS is a base64-encoded Storable string containing the POSTed data as +a hash ref. It normally contains at least one field, "uploaded files", +generated by /elements/file-upload.html and containing the list of uploaded +files. Currently only supports a single file named "file". + =cut use Storable qw(thaw); @@ -1331,26 +1431,23 @@ sub process_batch_import { my $dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc. '/'; my $file = $dir. $files{'file'}; - my $type; - if ( $file =~ /\.(\w+)$/i ) { - $type = lc($1); - } else { - #or error out??? - warn "can't parse file type from filename $file; defaulting to CSV"; - $type = 'csv'; - } - $type = 'csv' - if $opt->{'default_csv'} && $type ne 'xls'; - my $error = FS::Record::batch_import( { - table => $table, - formats => \%formats, - job => $job, - file => $file, - type => $type, - format => $param->{format}, - params => { map { $_ => $param->{$_} } @pass_params }, + #class-static + table => $table, + formats => \%formats, + format_types => $opt->{format_types}, + format_headers => $opt->{format_headers}, + format_sep_chars => $opt->{format_sep_chars}, + format_fixedlength_formats => $opt->{format_fixedlength_formats}, + #per-import + job => $job, + file => $file, + #type => $type, + format => $param->{format}, + params => { map { $_ => $param->{$_} } @pass_params }, + #? + default_csv => $opt->{default_csv}, } ); unlink $file; @@ -1358,31 +1455,115 @@ sub process_batch_import { die "$error\n" if $error; } +=item batch_import PARAM_HASHREF + +Class method for batch imports. Available params: + +=over 4 + +=item table + +=item formats + +=item format_types + +=item format_headers + +=item format_sep_chars + +=item format_fixedlength_formats + +=item params + +=item job + +FS::queue object, will be updated with progress + +=item file + +=item type + +csv, xls or fixedlength + +=item format + +=item empty_ok + +=back + +=cut + sub batch_import { my $param = shift; - my $table = $param->{table}; - my $formats = $param->{formats}; - my $params = $param->{params}; - - my $job = $param->{job}; + warn "$me batch_import call with params: \n". Dumper($param) + if $DEBUG; - my $filename = $param->{file}; - my $type = $param->{type} || 'csv'; + my $table = $param->{table}; + my $formats = $param->{formats}; - my $format = $param->{'format'}; + my $job = $param->{job}; + my $file = $param->{file}; + my $format = $param->{'format'}; + my $params = $param->{params} || {}; die "unknown format $format" unless exists $formats->{ $format }; - my @fields = @{ $formats->{ $format } }; + my $type = $param->{'format_types'} + ? $param->{'format_types'}{ $format } + : $param->{type} || 'csv'; + + unless ( $type ) { + if ( $file =~ /\.(\w+)$/i ) { + $type = lc($1); + } else { + #or error out??? + warn "can't parse file type from filename $file; defaulting to CSV"; + $type = 'csv'; + } + $type = 'csv' + if $param->{'default_csv'} && $type ne 'xls'; + } + + my $header = $param->{'format_headers'} + ? $param->{'format_headers'}{ $param->{'format'} } + : 0; + + my $sep_char = $param->{'format_sep_chars'} + ? $param->{'format_sep_chars'}{ $param->{'format'} } + : ','; + + my $fixedlength_format = + $param->{'format_fixedlength_formats'} + ? $param->{'format_fixedlength_formats'}{ $param->{'format'} } + : ''; + + my @fields = @{ $formats->{ $format } }; + + my $row = 0; my $count; my $parser; my @buffer = (); - if ( $type eq 'csv' ) { + if ( $type eq 'csv' || $type eq 'fixedlength' ) { + + if ( $type eq 'csv' ) { + + my %attr = (); + $attr{sep_char} = $sep_char if $sep_char; + $parser = new Text::CSV_XS \%attr; - $parser = new Text::CSV_XS; + } elsif ( $type eq 'fixedlength' ) { - @buffer = split(/\r?\n/, slurp($filename) ); + eval "use Parse::FixedLength;"; + die $@ if $@; + $parser = new Parse::FixedLength $fixedlength_format; + + } else { + die "Unknown file type $type\n"; + } + + @buffer = split(/\r?\n/, slurp($file) ); + splice(@buffer, 0, ($header || 0) ); $count = scalar(@buffer); } elsif ( $type eq 'xls' ) { @@ -1390,13 +1571,20 @@ sub batch_import { eval "use Spreadsheet::ParseExcel;"; die $@ if $@; - my $excel = Spreadsheet::ParseExcel::Workbook->new->Parse($filename); + eval "use DateTime::Format::Excel;"; + #for now, just let the error be thrown if it is used, since only CDR + # formats bill_west and troop use it, not other excel-parsing things + #die $@ if $@; + + my $excel = Spreadsheet::ParseExcel::Workbook->new->Parse($file); $parser = $excel->{Worksheet}[0]; #first sheet $count = $parser->{MaxRow} || $parser->{MinRow}; $count++; + $row = $header || 0; + } else { die "Unknown file type $type\n"; } @@ -1415,7 +1603,7 @@ sub batch_import { my $dbh = dbh; my $line; - my $row = 0; + my $imported = 0; my( $last, $min_sec ) = ( time, 5 ); #progressbar foo while (1) { @@ -1431,6 +1619,10 @@ sub batch_import { }; @columns = $parser->fields(); + } elsif ( $type eq 'fixedlength' ) { + + @columns = $parser->parse($line); + } elsif ( $type eq 'xls' ) { last if $row > ($parser->{MaxRow} || $parser->{MinRow}) @@ -1446,6 +1638,7 @@ sub batch_import { die "Unknown file type $type\n"; } + my @later = (); my %hash = %$params; foreach my $field ( @fields ) { @@ -1453,9 +1646,11 @@ sub batch_import { my $value = shift @columns; if ( ref($field) eq 'CODE' ) { - &{$field}(\%hash, $value); + #&{$field}(\%hash, $value); + push @later, $field, $value; } else { - $hash{$field} = $value if length($value); + #??? $hash{$field} = $value if length($value); + $hash{$field} = $value if defined($value) && length($value); } } @@ -1464,6 +1659,12 @@ sub batch_import { my $record = $class->new( \%hash ); + while ( scalar(@later) ) { + my $sub = shift @later; + my $data = shift @later; + &{$sub}($record, $data); # $record->&{$sub}($data); + } + my $error = $record->insert; if ( $error ) { @@ -1472,9 +1673,10 @@ sub batch_import { } $row++; + $imported++; if ( $job && time - $min_sec > $last ) { #progress bar - $job->update_statustext( int(100 * $row / $count) ); + $job->update_statustext( int(100 * $imported / $count) ); $last = time; } @@ -1482,7 +1684,7 @@ sub batch_import { $dbh->commit or die $dbh->errstr if $oldAutoCommit;; - return "Empty file!" unless $row; + return "Empty file!" unless $imported || $param->{empty_ok}; ''; #no error