package FS::tax_rate;
+use base qw( FS::Record );
use strict;
-use vars qw( @ISA $DEBUG $me
+use vars qw( $DEBUG $me
%tax_unittypes %tax_maxtypes %tax_basetypes %tax_authorities
- %tax_passtypes %GetInfoType );
+ %tax_passtypes %GetInfoType $keep_cch_files );
use Date::Parse;
use DateTime;
use DateTime::Format::Strptime;
-use Storable qw( thaw );
+use Storable qw( thaw nfreeze );
use IO::File;
use File::Temp;
+use Text::CSV_XS;
+use URI::Escape;
use LWP::UserAgent;
use HTTP::Request;
use HTTP::Response;
use DBIx::DBSchema::Table;
use DBIx::DBSchema::Column;
use FS::Record qw( qsearch qsearchs dbh dbdef );
+use FS::Conf;
use FS::tax_class;
use FS::cust_bill_pkg;
use FS::cust_tax_location;
use FS::cust_main;
use FS::Misc qw( csv_from_fixed );
-@ISA = qw( FS::Record );
-
$DEBUG = 0;
$me = '[FS::tax_rate]';
+$keep_cch_files = 0;
=head1 NAME
$self->ut_numbern('taxnum')
|| $self->ut_text('geocode')
|| $self->ut_textn('data_vendor')
- || $self->ut_textn('location')
+ || $self->ut_cch_textn('location')
|| $self->ut_foreign_key('taxclassnum', 'tax_class', 'taxclassnum')
|| $self->ut_snumbern('effective_date')
|| $self->ut_float('tax')
}
+#ut_text / ut_textn w/ ` added cause now that's in the data
+sub ut_cch_textn {
+ my($self,$field)=@_;
+ $self->getfield($field)
+ =~ /^([\wô \!\@\#\$\%\&\(\)\-\+\;\:\'\"\,\.\?\/\=\[\]\<\>\`]*)$/
+ or return gettext('illegal_or_empty_text'). " $field: ".
+ $self->getfield($field);
+ $self->setfield($field,$1);
+ '';
+
+}
+
=item taxclass_description
Returns the human understandable value associated with the related
$tax_passtypes{$self->passtype};
}
-=item taxline TAXABLES, [ OPTIONSHASH ]
+=item taxline TAXABLES
Returns a listref of a name and an amount of tax calculated for the list
of packages/amounts referenced by TAXABLES. If an error occurs, a message
sub taxline {
my $self = shift;
+ # this used to accept a hash of options but none of them did anything
+ # so it's been removed.
my $taxables;
- my %opt = ();
if (ref($_[0]) eq 'ARRAY') {
$taxables = shift;
- %opt = @_;
}else{
$taxables = [ @_ ];
#exemptions would be broken in this case
}
my $maxtype = $self->maxtype || 0;
- if ($maxtype != 0 && $maxtype != 9) {
+ if ($maxtype != 0 && $maxtype != 1 && $maxtype != 9) {
return $self->_fatal_or_null( 'tax with "'.
$self->maxtype_name. '" threshold'
);
my $taxable_units = 0;
unless ($self->recurtax =~ /^Y$/i) {
- if (( $self->unittype || 0 ) == 0) {
+
+ if (( $self->unittype || 0 ) == 0) { #access line
my %seen = ();
foreach (@cust_bill_pkg) {
$taxable_units += $_->units
- unless $seen{$_->pkgnum};
- $seen{$_->pkgnum}++;
+ unless $seen{$_->pkgnum}++;
}
- }elsif ($self->unittype == 1) {
+
+ } elsif ($self->unittype == 1) { #minute
return $self->_fatal_or_null( 'fee with minute unit type' );
- }elsif ($self->unittype == 2) {
- $taxable_units = 1;
- }else {
+
+ } elsif ($self->unittype == 2) { #account
+
+ my $conf = new FS::Conf;
+ if ( $conf->exists('tax-pkg_address') ) {
+ #number of distinct locations
+ my %seen = ();
+ foreach (@cust_bill_pkg) {
+ $taxable_units++
+ unless $seen{$_->cust_pkg->locationnum}++;
+ }
+ } else {
+ $taxable_units = 1;
+ }
+
+ } else {
return $self->_fatal_or_null( 'unknown unit type in tax'. $self->taxnum );
}
+
}
- #
- # XXX insert exemption handling here
+ # XXX handle excessrate (use_excessrate) / excessfee /
+ # taxbase/feebase / taxmax/feemax
+ # and eventually exemptions
#
# the tax or fee is applied to taxbase or feebase and then
# the excessrate or excess fee is applied to taxmax or feemax
- #
$amount += $taxable_charged * $self->tax;
$amount += $taxable_units * $self->fee;
my $conf = new FS::Conf;
- $error = "can't yet handle $error";
+ $error = "can't yet handle ". $error;
my $name = $self->taxname;
$name = 'Other surcharges'
if ($self->passtype == 2);
}
}
-=item tax_on_tax CUST_MAIN
+=item tax_on_tax CUST_LOCATION
Returns a list of taxes which are candidates for taxing taxes for the
-given customer (see L<FS::cust_main>)
+given service location (see L<FS::cust_location>)
=cut
+ #hot
sub tax_on_tax {
+ #akshun
my $self = shift;
- my $cust_main = shift;
+ my $cust_location = shift;
warn "looking up taxes on tax ". $self->taxnum. " for customer ".
- $cust_main->custnum
+ $cust_location->custnum
if $DEBUG;
- my $geocode = $cust_main->geocode($self->data_vendor);
+ my $geocode = $cust_location->geocode($self->data_vendor);
# CCH oddness in m2m
my $dbh = dbh;
=cut
+sub _progressbar_foo {
+ return (0, time, 5);
+}
+
sub batch_import {
my ($param, $job) = @_;
}
my $line;
- my ( $count, $last, $min_sec ) = (0, time, 5); #progressbar
+ my ( $count, $last, $min_sec ) = _progressbar_foo();
if ( $job || scalar(@column_callbacks) ) {
my $error =
csv_from_fixed(\$fh, \$count, \@column_lengths, \@column_callbacks);
$count *=2;
if ( $format eq 'cch' || $format eq 'cch-update' ) {
+ #false laziness w/below (sub _perform_cch_diff)
@fields = qw( geocode inoutcity inoutlocal tax location taxbase taxmax
excessrate effective_date taxauth taxtype taxcat taxname
usetax useexcessrate fee unittype feemax maxtype passflag
my $dt = $parser->parse_datetime( $hash->{'effective_date'} );
$hash->{'effective_date'} = $dt ? $dt->epoch : '';
+ $hash->{$_} =~ s/\s//g foreach qw( inoutcity inoutlocal ) ;
$hash->{$_} = sprintf("%.2f", $hash->{$_}) foreach qw( taxbase taxmax );
my $taxclassid =
die "unknown format $format";
}
- eval "use Text::CSV_XS;";
- die $@ if $@;
-
my $csv = new Text::CSV_XS;
my $imported = 0;
foreach my $field ( @fields ) {
$tax_rate{$field} = shift @columns;
}
+
if ( scalar( @columns ) ) {
$dbh->rollback if $oldAutoCommit;
- return "Unexpected trailing columns in line (wrong format?): $line";
+ return "Unexpected trailing columns in line (wrong format?) importing tax_rate: $line";
}
my $error = &{$hook}(\%tax_rate);
}
- for (grep { !exists($delete{$_}) } keys %insert) {
+ my @replace = grep { exists($delete{$_}) } keys %insert;
+ for (@replace) {
if ( $job ) { # progress bar
if ( time - $min_sec > $last ) {
my $error = $job->update_statustext(
}
}
- my $tax_rate = new FS::tax_rate( $insert{$_} );
- my $error = $tax_rate->insert;
+ my $old = qsearchs( 'tax_rate', $delete{$_} );
- if ( $error ) {
- $dbh->rollback if $oldAutoCommit;
- my $hashref = $insert{$_};
- $line = join(", ", map { "$_ => ". $hashref->{$_} } keys(%$hashref) );
- return "can't insert tax_rate for $line: $error";
+ if ( $old ) {
+
+ my $new = new FS::tax_rate({ $old->hash, %{$insert{$_}}, 'manual' => '' });
+ $new->taxnum($old->taxnum);
+ my $error = $new->replace($old);
+
+ if ( $error ) {
+ $dbh->rollback if $oldAutoCommit;
+ my $hashref = $insert{$_};
+ $line = join(", ", map { "$_ => ". $hashref->{$_} } keys(%$hashref) );
+ return "can't replace tax_rate for $line: $error";
+ }
+
+ $imported++;
+
+ } else {
+
+ $old = delete $delete{$_};
+ warn "WARNING: can't find tax_rate to replace (inserting instead and continuing) for: ".
+ #join(" ", map { "$_ => ". $old->{$_} } @fields);
+ join(" ", map { "$_ => ". $old->{$_} } keys(%$old) );
}
$imported++;
}
- for (grep { exists($delete{$_}) } keys %insert) {
+ for (grep { !exists($delete{$_}) } keys %insert) {
if ( $job ) { # progress bar
if ( time - $min_sec > $last ) {
my $error = $job->update_statustext(
}
}
- my $old = qsearchs( 'tax_rate', $delete{$_} );
- unless ($old) {
- $dbh->rollback if $oldAutoCommit;
- $old = $delete{$_};
- return "can't find tax_rate to replace for: ".
- #join(" ", map { "$_ => ". $old->{$_} } @fields);
- join(" ", map { "$_ => ". $old->{$_} } keys(%$old) );
- }
- my $new = new FS::tax_rate({ $old->hash, %{$insert{$_}}, 'manual' => '' });
- $new->taxnum($old->taxnum);
- my $error = $new->replace($old);
+ my $tax_rate = new FS::tax_rate( $insert{$_} );
+ my $error = $tax_rate->insert;
if ( $error ) {
$dbh->rollback if $oldAutoCommit;
my $hashref = $insert{$_};
$line = join(", ", map { "$_ => ". $hashref->{$_} } keys(%$hashref) );
- return "can't replace tax_rate for $line: $error";
+ return "can't insert tax_rate for $line: $error";
}
$imported++;
- $imported++;
}
for (grep { !exists($insert{$_}) } keys %delete) {
}
my $tax_rate = qsearchs( 'tax_rate', $delete{$_} );
- unless ($tax_rate) {
+ if (!$tax_rate) {
$dbh->rollback if $oldAutoCommit;
$tax_rate = $delete{$_};
- return "can't find tax_rate to delete for: ".
- #join(" ", map { "$_ => ". $tax_rate->{$_} } @fields);
- join(" ", map { "$_ => ". $tax_rate->{$_} } keys(%$tax_rate) );
- }
- my $error = $tax_rate->delete;
+ warn "WARNING: can't find tax_rate to delete for: ".
+ join(" ", map { "$_ => ". $tax_rate->{$_} } keys(%$tax_rate) ).
+ " (ignoring)\n";
+ } else {
+ my $error = $tax_rate->delete; # XXX we really should not do this
+ # (it orphans CBPTRL records)
- if ( $error ) {
- $dbh->rollback if $oldAutoCommit;
- my $hashref = $delete{$_};
- $line = join(", ", map { "$_ => ". $hashref->{$_} } keys(%$hashref) );
- return "can't delete tax_rate for $line: $error";
+ if ( $error ) {
+ $dbh->rollback if $oldAutoCommit;
+ my $hashref = $delete{$_};
+ $line = join(", ", map { "$_ => ". $hashref->{$_} } keys(%$hashref) );
+ return "can't delete tax_rate for $line: $error";
+ }
}
$imported++;
sub process_batch_import {
my $job = shift;
+ my $oldAutoCommit = $FS::UID::AutoCommit;
+ local $FS::UID::AutoCommit = 0;
+ my $dbh = dbh;
+
+ my $param = thaw(decode_base64(shift));
+ my $args = '$job, encode_base64( nfreeze( $param ) )';
+
+ my $method = '_perform_batch_import';
+ if ( $param->{reload} ) {
+ $method = 'process_batch_reload';
+ }
+
+ eval "$method($args);";
+ if ($@) {
+ $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
+ die $@;
+ }
+
+ #success!
+ $dbh->commit or die $dbh->errstr if $oldAutoCommit;
+}
+
+sub _perform_batch_import {
+ my $job = shift;
+
my $param = thaw(decode_base64(shift));
my $format = $param->{'format'}; #well... this is all cch specific
my $files = $param->{'uploaded_files'}
or die "No files provided.";
- my (%files) = map { /^(\w+):([\.\w]+)$/ ? ($1,$2):() } split /,/, $files;
-
- if ($format eq 'cch' || $format eq 'cch-fixed') {
-
- my $oldAutoCommit = $FS::UID::AutoCommit;
- local $FS::UID::AutoCommit = 0;
- my $dbh = dbh;
- my $error = '';
- my $have_location = 0;
-
- my @list = ( 'GEOCODE', 'geofile', \&FS::tax_rate_location::batch_import,
- 'CODE', 'codefile', \&FS::tax_class::batch_import,
- 'PLUS4', 'plus4file', \&FS::cust_tax_location::batch_import,
- 'ZIP', 'zipfile', \&FS::cust_tax_location::batch_import,
- 'TXMATRIX', 'txmatrix', \&FS::part_pkg_taxrate::batch_import,
- 'DETAIL', 'detail', \&FS::tax_rate::batch_import,
- );
- while( scalar(@list) ) {
- my ($name, $file, $import_sub) = (shift @list, shift @list, shift @list);
- unless ($files{$file}) {
- next if $name eq 'PLUS4';
- $error = "No $name supplied";
- $error = "Neither PLUS4 nor ZIP supplied"
- if ($name eq 'ZIP' && !$have_location);
- next;
- }
- $have_location = 1 if $name eq 'PLUS4';
- my $fmt = $format. ( $name eq 'ZIP' ? '-zip' : '' );
- my $dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc;
- my $filename = "$dir/". $files{$file};
- open my $fh, "< $filename" or $error ||= "Can't open $name file: $!";
-
- $error ||= &{$import_sub}({ 'filehandle' => $fh, 'format' => $fmt }, $job);
- close $fh;
- unlink $filename or warn "Can't delete $filename: $!";
- }
-
- if ($error) {
- $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
- die $error;
- }else{
- $dbh->commit or die $dbh->errstr if $oldAutoCommit;
- }
+ my (%files) = map { /^(\w+):((taxdata\/\w+\.\w+\/)?[\.\w]+)$/ ? ($1,$2):() }
+ split /,/, $files;
- }elsif ($format eq 'cch-update' || $format eq 'cch-fixed-update') {
+ if ( $format eq 'cch' || $format eq 'cch-fixed'
+ || $format eq 'cch-update' || $format eq 'cch-fixed-update' )
+ {
my $oldAutoCommit = $FS::UID::AutoCommit;
local $FS::UID::AutoCommit = 0;
my @insert_list = ();
my @delete_list = ();
my @predelete_list = ();
+ my $insertname = '';
+ my $deletename = '';
+ my $dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc;
- my @list = ( 'GEOCODE', 'geofile', \&FS::tax_rate_location::batch_import,
- 'CODE', 'codefile', \&FS::tax_class::batch_import,
- 'PLUS4', 'plus4file', \&FS::cust_tax_location::batch_import,
- 'ZIP', 'zipfile', \&FS::cust_tax_location::batch_import,
- 'TXMATRIX', 'txmatrix', \&FS::part_pkg_taxrate::batch_import,
+ my @list = ( 'GEOCODE', \&FS::tax_rate_location::batch_import,
+ 'CODE', \&FS::tax_class::batch_import,
+ 'PLUS4', \&FS::cust_tax_location::batch_import,
+ 'ZIP', \&FS::cust_tax_location::batch_import,
+ 'TXMATRIX', \&FS::part_pkg_taxrate::batch_import,
+ 'DETAIL', \&FS::tax_rate::batch_import,
);
- my $dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc;
while( scalar(@list) ) {
- my ($name, $file, $import_sub) = (shift @list, shift @list, shift @list);
+ my ( $name, $import_sub ) = splice( @list, 0, 2 );
+ my $file = lc($name). 'file';
+
unless ($files{$file}) {
- my $vendor = $name eq 'ZIP' ? 'cch' : 'cch-zip';
- next # update expected only for previously installed location data
- if ( ($name eq 'PLUS4' || $name eq 'ZIP')
- && !scalar( qsearch( { table => 'cust_tax_location',
- hashref => { data_vendor => $vendor },
- select => 'DISTINCT data_vendor',
- } )
- )
- );
-
- $error = "No $name supplied";
+ #$error = "No $name supplied";
next;
}
+ next if $name eq 'DETAIL' && $format =~ /update/;
+
my $filename = "$dir/". $files{$file};
- open my $fh, "< $filename" or $error ||= "Can't open $name file $filename: $!";
- unlink $filename or warn "Can't delete $filename: $!";
-
- my $ifh = new File::Temp( TEMPLATE => "$name.insert.XXXXXXXX",
- DIR => $dir,
- UNLINK => 0, #meh
- ) or die "can't open temp file: $!\n";
-
- my $dfh = new File::Temp( TEMPLATE => "$name.delete.XXXXXXXX",
- DIR => $dir,
- UNLINK => 0, #meh
- ) or die "can't open temp file: $!\n";
-
- my $insert_pattern = ($format eq 'cch-update') ? qr/"I"\s*$/ : qr/I\s*$/;
- my $delete_pattern = ($format eq 'cch-update') ? qr/"D"\s*$/ : qr/D\s*$/;
- while(<$fh>) {
- my $handle = '';
- $handle = $ifh if $_ =~ /$insert_pattern/;
- $handle = $dfh if $_ =~ /$delete_pattern/;
- unless ($handle) {
- $error = "bad input line: $_" unless $handle;
- last;
+
+ if ( $format =~ /update/ ) {
+
+ ( $error, $insertname, $deletename ) =
+ _perform_cch_insert_delete_split( $name, $filename, $dir, $format )
+ unless $error;
+ last if $error;
+
+ unlink $filename or warn "Can't delete $filename: $!"
+ unless $keep_cch_files;
+ push @insert_list, $name, $insertname, $import_sub, $format;
+ if ( $name eq 'GEOCODE' || $name eq 'CODE' ) { #handle this whole ordering issue better
+ unshift @predelete_list, $name, $deletename, $import_sub, $format;
+ } else {
+ unshift @delete_list, $name, $deletename, $import_sub, $format;
}
- print $handle $_;
- }
- close $fh;
- close $ifh;
- close $dfh;
- push @insert_list, $name, $ifh->filename, $import_sub;
- if ( $name eq 'GEOCODE' ) { #handle this whole ordering issue better
- unshift @predelete_list, $name, $dfh->filename, $import_sub;
} else {
- unshift @delete_list, $name, $dfh->filename, $import_sub;
+
+ push @insert_list, $name, $filename, $import_sub, $format;
+
}
}
- while( scalar(@predelete_list) ) {
- my ($name, $file, $import_sub) =
- (shift @predelete_list, shift @predelete_list, shift @predelete_list);
+ push @insert_list,
+ 'DETAIL', "$dir/".$files{detailfile}, \&FS::tax_rate::batch_import, $format
+ if $format =~ /update/;
- my $fmt = $format. ( $name eq 'ZIP' ? '-zip' : '' );
- open my $fh, "< $file" or $error ||= "Can't open $name file $file: $!";
- $error ||=
- &{$import_sub}({ 'filehandle' => $fh, 'format' => $fmt }, $job);
- close $fh;
- unlink $file or warn "Can't delete $file: $!";
+ my %addl_param = ();
+ if ( $param->{'delete_only'} ) {
+ $addl_param{'delete_only'} = $param->{'delete_only'};
+ @insert_list = ()
}
-
- while( scalar(@insert_list) ) {
- my ($name, $file, $import_sub) =
- (shift @insert_list, shift @insert_list, shift @insert_list);
- my $fmt = $format. ( $name eq 'ZIP' ? '-zip' : '' );
- open my $fh, "< $file" or $error ||= "Can't open $name file $file: $!";
- $error ||=
- &{$import_sub}({ 'filehandle' => $fh, 'format' => $fmt }, $job);
- close $fh;
- unlink $file or warn "Can't delete $file: $!";
- }
+ $error ||= _perform_cch_tax_import( $job,
+ [ @predelete_list ],
+ [ @insert_list ],
+ [ @delete_list ],
+ \%addl_param,
+ );
- $error ||= "No DETAIL supplied"
- unless ($files{detail});
- open my $fh, "< $dir/". $files{detail}
- or $error ||= "Can't open DETAIL file: $!";
- $error ||=
- &FS::tax_rate::batch_import({ 'filehandle' => $fh, 'format' => $format },
- $job);
- close $fh;
- unlink "$dir/". $files{detail} or warn "Can't delete $files{detail}: $!"
- if $files{detail};
-
- while( scalar(@delete_list) ) {
- my ($name, $file, $import_sub) =
- (shift @delete_list, shift @delete_list, shift @delete_list);
-
- my $fmt = $format. ( $name eq 'ZIP' ? '-zip' : '' );
- open my $fh, "< $file" or $error ||= "Can't open $name file $file: $!";
- $error ||=
- &{$import_sub}({ 'filehandle' => $fh, 'format' => $fmt }, $job);
- close $fh;
+
+ @list = ( @predelete_list, @insert_list, @delete_list );
+ while( !$keep_cch_files && scalar(@list) ) {
+ my ( undef, $file, undef, undef ) = splice( @list, 0, 4 );
unlink $file or warn "Can't delete $file: $!";
}
-
+
if ($error) {
$dbh->rollback or die $dbh->errstr if $oldAutoCommit;
die $error;
}
-=item process_download_and_reload
-Download and process a tax update as a queued JSRPC job after wiping the
-existing wipable tax data.
+sub _perform_cch_tax_import {
+ my ( $job, $predelete_list, $insert_list, $delete_list, $addl_param ) = @_;
+ $addl_param ||= {};
-=cut
+ my $error = '';
+ foreach my $list ($predelete_list, $insert_list, $delete_list) {
+ while( scalar(@$list) ) {
+ my ( $name, $file, $method, $format ) = splice( @$list, 0, 4 );
+ my $fmt = "$format-update";
+ $fmt = $format. ( lc($name) eq 'zip' ? '-zip' : '' );
+ open my $fh, "< $file" or $error ||= "Can't open $name file $file: $!";
+ my $param = { 'filehandle' => $fh,
+ 'format' => $fmt,
+ %$addl_param,
+ };
+ $error ||= &{$method}($param, $job);
+ close $fh;
+ }
+ }
-sub process_download_and_reload {
- my $job = shift;
+ return $error;
+}
- my $param = thaw(decode_base64($_[0]));
- my $format = $param->{'format'}; #well... this is all cch specific
+sub _perform_cch_insert_delete_split {
+ my ($name, $filename, $dir, $format) = @_;
- my ( $count, $last, $min_sec, $imported ) = (0, time, 5, 0); #progressbar
- $count = 100;
+ my $error = '';
- if ( $job ) { # progress bar
- my $error = $job->update_statustext( int( 100 * $imported / $count ) );
+ open my $fh, "< $filename"
+ or $error ||= "Can't open $name file $filename: $!";
+
+ my $ifh = new File::Temp( TEMPLATE => "$name.insert.XXXXXXXX",
+ DIR => $dir,
+ UNLINK => 0, #meh
+ ) or die "can't open temp file: $!\n";
+ my $insertname = $ifh->filename;
+
+ my $dfh = new File::Temp( TEMPLATE => "$name.delete.XXXXXXXX",
+ DIR => $dir,
+ UNLINK => 0, #meh
+ ) or die "can't open temp file: $!\n";
+ my $deletename = $dfh->filename;
+
+ my $insert_pattern = ($format eq 'cch-update') ? qr/"I"\s*$/ : qr/I\s*$/;
+ my $delete_pattern = ($format eq 'cch-update') ? qr/"D"\s*$/ : qr/D\s*$/;
+ while(<$fh>) {
+ my $handle = '';
+ $handle = $ifh if $_ =~ /$insert_pattern/;
+ $handle = $dfh if $_ =~ /$delete_pattern/;
+ unless ($handle) {
+ $error = "bad input line: $_" unless $handle;
+ last;
+ }
+ print $handle $_;
+ }
+ close $fh;
+ close $ifh;
+ close $dfh;
+
+ return ($error, $insertname, $deletename);
+}
+
+sub _perform_cch_diff {
+ my ($name, $newdir, $olddir) = @_;
+
+ my %oldlines = ();
+
+ if ($olddir) {
+ open my $oldcsvfh, "$olddir/$name.txt"
+ or die "failed to open $olddir/$name.txt: $!\n";
+
+ while(<$oldcsvfh>) {
+ chomp;
+ $oldlines{$_} = 1;
+ }
+ close $oldcsvfh;
+ }
+
+ open my $newcsvfh, "$newdir/$name.txt"
+ or die "failed to open $newdir/$name.txt: $!\n";
+
+ my $dfh = new File::Temp( TEMPLATE => "$name.diff.XXXXXXXX",
+ DIR => "$newdir",
+ UNLINK => 0, #meh
+ ) or die "can't open temp file: $!\n";
+ my $diffname = $dfh->filename;
+
+ while(<$newcsvfh>) {
+ chomp;
+ if (exists($oldlines{$_})) {
+ $oldlines{$_} = 0;
+ } else {
+ print $dfh $_, ',"I"', "\n";
+ }
+ }
+ close $newcsvfh;
+
+ #false laziness w/above (sub batch_import)
+ my @fields = qw( geocode inoutcity inoutlocal tax location taxbase taxmax
+ excessrate effective_date taxauth taxtype taxcat taxname
+ usetax useexcessrate fee unittype feemax maxtype passflag
+ passtype basetype );
+ my $numfields = scalar(@fields);
+
+ my $csv = new Text::CSV_XS { 'always_quote' => 1 };
+
+ for my $line (grep $oldlines{$_}, keys %oldlines) {
+
+ $csv->parse($line) or do {
+ #$dbh->rollback if $oldAutoCommit;
+ die "can't parse: ". $csv->error_input();
+ };
+ my @columns = $csv->fields();
+
+ $csv->combine( splice(@columns, 0, $numfields) );
+
+ print $dfh $csv->string, ',"D"', "\n";
+ }
+
+ close $dfh;
+
+ return $diffname;
+}
+
+sub _cch_fetch_and_unzip {
+ my ( $job, $urls, $secret, $dir ) = @_;
+
+ my $ua = new LWP::UserAgent;
+ foreach my $url (split ',', $urls) {
+ my @name = split '/', $url; #somewhat restrictive
+ my $name = pop @name;
+ $name =~ /([\w.]+)/; # untaint that which we don't trust so much any more
+ $name = $1;
+
+ open my $taxfh, ">$dir/$name" or die "Can't open $dir/$name: $!\n";
+
+ my ( $imported, $last, $min_sec ) = _progressbar_foo();
+ my $res = $ua->request(
+ new HTTP::Request( GET => $url ),
+ sub {
+ print $taxfh $_[0] or die "Can't write to $dir/$name: $!\n";
+ my $content_length = $_[1]->content_length;
+ $imported += length($_[0]);
+ if ( time - $min_sec > $last ) {
+ my $error = $job->update_statustext(
+ ($content_length ? int(100 * $imported/$content_length) : 0 ).
+ ",Downloading data from CCH"
+ );
+ die $error if $error;
+ $last = time;
+ }
+ },
+ );
+ die "download of $url failed: ". $res->status_line
+ unless $res->is_success;
+
+ close $taxfh;
+ my $error = $job->update_statustext( "0,Unpacking data" );
die $error if $error;
+ $secret =~ /([\w.]+)/; # untaint that which we don't trust so much any more
+ $secret = $1;
+ system('unzip', "-P", $secret, "-d", "$dir", "$dir/$name") == 0
+ or die "unzip -P $secret -d $dir $dir/$name failed";
+ #unlink "$dir/$name";
}
+}
+
+sub _cch_extract_csv_from_dbf {
+ my ( $job, $dir, $name ) = @_;
- my $oldAutoCommit = $FS::UID::AutoCommit;
- local $FS::UID::AutoCommit = 0;
- my $dbh = dbh;
- my $error = '';
+ eval "use XBase;";
+ die $@ if $@;
- my $sql =
- "SELECT count(*) FROM part_pkg_taxoverride JOIN tax_class ".
- "USING (taxclassnum) WHERE data_vendor = '$format'";
- my $sth = $dbh->prepare($sql) or die $dbh->errstr;
- $sth->execute
- or die "Unexpected error executing statement $sql: ". $sth->errstr;
- die "Don't (yet) know how to handle part_pkg_taxoverride records."
- if $sth->fetchrow_arrayref->[0];
+ my ( $imported, $last, $min_sec ) = _progressbar_foo();
+ my $error = $job->update_statustext( "0,Unpacking $name" );
+ die $error if $error;
+ warn "opening $dir.new/$name.dbf\n" if $DEBUG;
+ my $table = new XBase 'name' => "$dir.new/$name.dbf";
+ die "failed to access $dir.new/$name.dbf: ". XBase->errstr
+ unless defined($table);
+ my $count = $table->last_record; # approximately;
+ open my $csvfh, ">$dir.new/$name.txt"
+ or die "failed to open $dir.new/$name.txt: $!\n";
+
+ my $csv = new Text::CSV_XS { 'always_quote' => 1 };
+ my @fields = $table->field_names;
+ my $cursor = $table->prepare_select;
+ my $format_date =
+ sub { my $date = shift;
+ $date =~ /^(\d{4})(\d{2})(\d{2})$/ && ($date = "$2/$3/$1");
+ $date;
+ };
+ while (my $row = $cursor->fetch_hashref) {
+ $csv->combine( map { my $type = $table->field_type($_);
+ if ($type eq 'D') {
+ &{$format_date}($row->{$_}) ;
+ } elsif ($type eq 'N' && $row->{$_} =~ /e-/i ) {
+ sprintf('%.8f', $row->{$_}); #db row is numeric(14,8)
+ } else {
+ $row->{$_};
+ }
+ }
+ @fields
+ );
+ print $csvfh $csv->string, "\n";
+ $imported++;
+ if ( time - $min_sec > $last ) {
+ my $error = $job->update_statustext(
+ int(100 * $imported/$count). ",Unpacking $name"
+ );
+ die $error if $error;
+ $last = time;
+ }
+ }
+ $table->close;
+ close $csvfh;
+}
- # really should get a table EXCLUSIVE lock here
+sub _remember_disabled_taxes {
+ my ( $job, $format, $disabled_tax_rate ) = @_;
+
+ # cch specific hash
+
+ my ( $imported, $last, $min_sec ) = _progressbar_foo();
- #remember disabled taxes
- my %disabled_tax_rate = ();
my @items = qsearch( { table => 'tax_rate',
hashref => { disabled => 'Y',
data_vendor => $format,
select => 'geocode, taxclassnum',
}
);
- $count = scalar(@items);
+ my $count = scalar(@items);
foreach my $tax_rate ( @items ) {
if ( time - $min_sec > $last ) {
- my $error = $job->update_statustext(
+ $job->update_statustext(
int( 100 * $imported / $count ). ",Remembering disabled taxes"
);
- if ($error) {
- $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
- die $error;
- }
$last = time;
}
$imported++;
warn "failed to find tax_class ". $tax_rate->taxclassnum;
next;
}
- $disabled_tax_rate{$tax_rate->geocode. ':'. $tax_class->taxclass} = 1;
+ $disabled_tax_rate->{$tax_rate->geocode. ':'. $tax_class->taxclass} = 1;
}
+}
+
+sub _remember_tax_products {
+ my ( $job, $format, $taxproduct ) = @_;
- #remember tax products
# XXX FIXME this loop only works when cch is the only data provider
- my %taxproduct = ();
+
+ my ( $imported, $last, $min_sec ) = _progressbar_foo();
+
my $extra_sql = "WHERE taxproductnum IS NOT NULL OR ".
"0 < ( SELECT count(*) from part_pkg_option WHERE ".
" part_pkg_option.pkgpart = part_pkg.pkgpart AND ".
" optionname LIKE 'usage_taxproductnum_%' AND ".
" optionvalue != '' )";
- @items = qsearch( { table => 'part_pkg',
- select => 'DISTINCT pkgpart,taxproductnum',
- hashref => {},
- extra_sql => $extra_sql,
- }
- );
- $count = scalar(@items);
- $imported = 0;
+ my @items = qsearch( { table => 'part_pkg',
+ select => 'DISTINCT pkgpart,taxproductnum',
+ hashref => {},
+ extra_sql => $extra_sql,
+ }
+ );
+ my $count = scalar(@items);
foreach my $part_pkg ( @items ) {
if ( time - $min_sec > $last ) {
- my $error = $job->update_statustext(
+ $job->update_statustext(
int( 100 * $imported / $count ). ",Remembering tax products"
);
- if ($error) {
- $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
- die $error;
- }
$last = time;
}
$imported++;
warn "working with package part ". $part_pkg->pkgpart.
"which has a taxproductnum of ". $part_pkg->taxproductnum. "\n" if $DEBUG;
my $part_pkg_taxproduct = $part_pkg->taxproduct('');
- $taxproduct{$part_pkg->pkgpart}{''} = $part_pkg_taxproduct->taxproduct
- if $part_pkg_taxproduct;
+ $taxproduct->{$part_pkg->pkgpart}->{''} = $part_pkg_taxproduct->taxproduct
+ if $part_pkg_taxproduct && $part_pkg_taxproduct->data_vendor eq $format;
foreach my $option ( $part_pkg->part_pkg_option ) {
- next unless $option->optionname =~ /^usage_taxproductnum_(\w)$/;
+ next unless $option->optionname =~ /^usage_taxproductnum_(\w+)$/;
my $class = $1;
$part_pkg_taxproduct = $part_pkg->taxproduct($class);
- $taxproduct{$part_pkg->pkgpart}{$class} = $part_pkg_taxproduct->taxproduct
- if $part_pkg_taxproduct;
+ $taxproduct->{$part_pkg->pkgpart}->{$class} =
+ $part_pkg_taxproduct->taxproduct
+ if $part_pkg_taxproduct && $part_pkg_taxproduct->data_vendor eq $format;
}
}
+}
- #wipe out the old data
- $error = $job->update_statustext( "0,Removing old tax data" );
- if ($error) {
- $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
- die $error;
- }
- foreach my $tax_rate_location ( qsearch( 'tax_rate_location',
- { data_vendor => $format,
- disabled => '',
- }
- )
- )
- {
- $tax_rate_location->disabled('Y');
- my $error = $tax_rate_location->replace;
- if ( $error ) {
- $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
- die $error;
- }
- }
+sub _restore_remembered_tax_products {
+ my ( $job, $format, $taxproduct ) = @_;
- local $FS::part_pkg_taxproduct::delete_kludge = 1;
- my @table = qw(
- tax_rate part_pkg_taxrate part_pkg_taxproduct tax_class cust_tax_location
- );
- foreach my $table ( @table ) {
- my $dbh = dbh;
-# my $primary_key = dbdef->table($table)->primary_key;
-# my $sql = "SELECT $primary_key FROM $table WHERE data_vendor = ".
- my $sql = "DELETE FROM $table WHERE data_vendor = ".
- $dbh->quote($format);
- my $sth = $dbh->prepare($sql);
- unless ($sth) {
- $error = $dbh->errstr;
- $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
- die $error;
- }
- unless ($sth->execute) {
- $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
- die "Failed to execute $sql: ". $sth->errstr;
+ # cch specific
+
+ my ( $imported, $last, $min_sec ) = _progressbar_foo();
+ my $count = scalar(keys %$taxproduct);
+ foreach my $pkgpart ( keys %$taxproduct ) {
+ warn "restoring taxproductnums on pkgpart $pkgpart\n" if $DEBUG;
+ if ( time - $min_sec > $last ) {
+ $job->update_statustext(
+ int( 100 * $imported / $count ). ",Restoring tax products"
+ );
+ $last = time;
}
-# foreach my $row ( @{ $sth->fetchall_arrayref } ) {
-# my $record = qsearchs( $table, { $primary_key => $row->[0] } )
-# or die "Failed to find $table with $primary_key ". $row->[0];
-# my $error = $record->delete;
-# if ( $error ) {
-# $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
-# die $error;
-# }
-# }
- }
-
- if ( $format eq 'cch' ) {
- foreach my $cust_tax_location ( qsearch( 'cust_tax_location',
- { data_vendor => "$format-zip" }
- )
- )
- {
- my $error = $cust_tax_location->delete;
- if ( $error ) {
- $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
- die $error;
- }
- }
- }
-
- #import new data
- my $statement = ' &process_download_and_update($job, @_); ';
- eval $statement;
- if ($@) {
- $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
- die $@;
- }
-
- #restore taxproducts
- $count = scalar(keys %taxproduct);
- $imported = 0;
- foreach my $pkgpart ( keys %taxproduct ) {
- warn "restoring taxproductnums on pkgpart $pkgpart\n" if $DEBUG;
- if ( time - $min_sec > $last ) {
- my $error = $job->update_statustext(
- int( 100 * $imported / $count ). ",Restoring tax products"
- );
- if ( $error ) {
- $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
- die $error;
- }
- $last = time;
- }
- $imported++;
+ $imported++;
my $part_pkg = qsearchs('part_pkg', { pkgpart => $pkgpart } );
unless ( $part_pkg ) {
- $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
- die "somehow failed to find part_pkg with pkgpart $pkgpart!\n";
+ return "somehow failed to find part_pkg with pkgpart $pkgpart!\n";
}
my %options = $part_pkg->options;
my $primary_svc = $part_pkg->svcpart;
my $new = new FS::part_pkg { $part_pkg->hash };
- foreach my $class ( keys %{ $taxproduct{$pkgpart} } ) {
+ foreach my $class ( keys %{ $taxproduct->{$pkgpart} } ) {
warn "working with class '$class'\n" if $DEBUG;
my $part_pkg_taxproduct =
qsearchs( 'part_pkg_taxproduct',
- { taxproduct => $taxproduct{$pkgpart}{$class},
+ { taxproduct => $taxproduct->{$pkgpart}->{$class},
data_vendor => $format,
}
);
unless ( $part_pkg_taxproduct ) {
- $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
- die "failed to find part_pkg_taxproduct ($taxproduct{pkgpart}{$class})".
- " for pkgpart $pkgpart\n";
+ return "failed to find part_pkg_taxproduct (".
+ $taxproduct->{$pkgpart}->{$class}. ") for pkgpart $pkgpart\n";
}
if ( $class eq '' ) {
'options' => \%options,
);
- if ( $error ) {
- $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
- die $error;
- }
+ return $error if $error;
+
}
- #disable tax_rates
- $count = scalar(keys %disabled_tax_rate);
- $imported = 0;
- foreach my $key (keys %disabled_tax_rate) {
+ '';
+}
+
+sub _restore_remembered_disabled_taxes {
+ my ( $job, $format, $disabled_tax_rate ) = @_;
+
+ my ( $imported, $last, $min_sec ) = _progressbar_foo();
+ my $count = scalar(keys %$disabled_tax_rate);
+ foreach my $key (keys %$disabled_tax_rate) {
if ( time - $min_sec > $last ) {
- my $error = $job->update_statustext(
+ $job->update_statustext(
int( 100 * $imported / $count ). ",Disabling tax rates"
);
- if ( $error ) {
- $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
- die $error;
- }
$last = time;
}
$imported++;
my @tax_class = qsearch( 'tax_class', { data_vendor => $format,
taxclass => $taxclass,
} );
- if (scalar(@tax_class) > 1) {
- $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
- die "found multiple tax_class records for format $format class $taxclass";
- }
+ return "found multiple tax_class records for format $format class $taxclass"
+ if scalar(@tax_class) > 1;
unless (scalar(@tax_class)) {
warn "no tax_class for format $format class $taxclass\n";
);
if (scalar(@tax_rate) > 1) {
- $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
- die "found multiple tax_rate records for format $format geocode $geocode".
- " and taxclass $taxclass ( taxclassnum ". $tax_class[0]->taxclassnum.
- " )";
+ return "found multiple tax_rate records for format $format geocode ".
+ "$geocode and taxclass $taxclass ( taxclassnum ".
+ $tax_class[0]->taxclassnum. " )";
}
if (scalar(@tax_rate)) {
$tax_rate[0]->disabled('Y');
my $error = $tax_rate[0]->replace;
- if ( $error ) {
- $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
- die $error;
- }
+ return $error if $error;
}
}
+}
- #success!
+sub _remove_old_tax_data {
+ my ( $job, $format ) = @_;
+
+ my $dbh = dbh;
+ my $error = $job->update_statustext( "0,Removing old tax data" );
+ die $error if $error;
+
+ my $sql = "UPDATE public.tax_rate_location SET disabled='Y' ".
+ "WHERE data_vendor = ". $dbh->quote($format);
+ $dbh->do($sql) or return "Failed to execute $sql: ". $dbh->errstr;
+
+ my @table = qw(
+ tax_rate part_pkg_taxrate part_pkg_taxproduct tax_class cust_tax_location
+ );
+ foreach my $table ( @table ) {
+ $sql = "DELETE FROM public.$table WHERE data_vendor = ".
+ $dbh->quote($format);
+ $dbh->do($sql) or return "Failed to execute $sql: ". $dbh->errstr;
+ }
+
+ if ( $format eq 'cch' ) {
+ $sql = "DELETE FROM public.cust_tax_location WHERE data_vendor = ".
+ $dbh->quote("$format-zip");
+ $dbh->do($sql) or return "Failed to execute $sql: ". $dbh->errstr;
+ }
+
+ '';
+}
+
+sub _create_temporary_tables {
+ my ( $job, $format ) = @_;
+
+ my $dbh = dbh;
+ my $error = $job->update_statustext( "0,Creating temporary tables" );
+ die $error if $error;
+
+ my @table = qw( tax_rate
+ tax_rate_location
+ part_pkg_taxrate
+ part_pkg_taxproduct
+ tax_class
+ cust_tax_location
+ );
+ foreach my $table ( @table ) {
+ my $sql =
+ "CREATE TEMPORARY TABLE $table ( LIKE $table INCLUDING DEFAULTS )";
+ $dbh->do($sql) or return "Failed to execute $sql: ". $dbh->errstr;
+ }
+
+ '';
+}
+
+sub _copy_from_temp {
+ my ( $job, $format ) = @_;
+
+ my $dbh = dbh;
+ my $error = $job->update_statustext( "0,Making permanent" );
+ die $error if $error;
+
+ my @table = qw( tax_rate
+ tax_rate_location
+ part_pkg_taxrate
+ part_pkg_taxproduct
+ tax_class
+ cust_tax_location
+ );
+ foreach my $table ( @table ) {
+ my $sql =
+ "INSERT INTO public.$table SELECT * from $table";
+ $dbh->do($sql) or return "Failed to execute $sql: ". $dbh->errstr;
+ }
+
+ '';
+}
+
+=item process_download_and_reload
+
+Download and process a tax update as a queued JSRPC job after wiping the
+existing wipable tax data.
+
+=cut
+
+sub process_download_and_reload {
+ _process_reload('process_download_and_update', @_);
+}
- $dbh->commit or die $dbh->errstr if $oldAutoCommit;
+=item process_batch_reload
+
+Load and process a tax update from the provided files as a queued JSRPC job
+after wiping the existing wipable tax data.
+
+=cut
+
+sub process_batch_reload {
+ _process_reload('_perform_batch_import', @_);
+}
+
+
+sub _process_reload {
+ my ( $method, $job ) = ( shift, shift );
+
+ my $param = thaw(decode_base64($_[0]));
+ my $format = $param->{'format'}; #well... this is all cch specific
+
+ my ( $imported, $last, $min_sec ) = _progressbar_foo();
+
+ if ( $job ) { # progress bar
+ my $error = $job->update_statustext( 0 );
+ die $error if $error;
+ }
+
+ my $oldAutoCommit = $FS::UID::AutoCommit;
+ local $FS::UID::AutoCommit = 0;
+ my $dbh = dbh;
+ my $error = '';
+
+ my $sql =
+ "SELECT count(*) FROM part_pkg_taxoverride JOIN tax_class ".
+ "USING (taxclassnum) WHERE data_vendor = '$format'";
+ my $sth = $dbh->prepare($sql) or die $dbh->errstr;
+ $sth->execute
+ or die "Unexpected error executing statement $sql: ". $sth->errstr;
+ die "Don't (yet) know how to handle part_pkg_taxoverride records."
+ if $sth->fetchrow_arrayref->[0];
+
+ # really should get a table EXCLUSIVE lock here
+
+ #remember disabled taxes
+ my %disabled_tax_rate = ();
+ $error ||= _remember_disabled_taxes( $job, $format, \%disabled_tax_rate );
+
+ #remember tax products
+ my %taxproduct = ();
+ $error ||= _remember_tax_products( $job, $format, \%taxproduct );
+
+ #create temp tables
+ $error ||= _create_temporary_tables( $job, $format );
+
+ #import new data
+ unless ($error) {
+ my $args = '$job, @_';
+ eval "$method($args);";
+ $error = $@ if $@;
+ }
+
+ #restore taxproducts
+ $error ||= _restore_remembered_tax_products( $job, $format, \%taxproduct );
+
+ #disable tax_rates
+ $error ||=
+ _restore_remembered_disabled_taxes( $job, $format, \%disabled_tax_rate );
+
+ #wipe out the old data
+ $error ||= _remove_old_tax_data( $job, $format );
+
+ #untemporize
+ $error ||= _copy_from_temp( $job, $format );
+
+ if ($error) {
+ $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
+ die $error;
+ }
+
+ #success!
+ $dbh->commit or die $dbh->errstr if $oldAutoCommit;
}
+
=item process_download_and_update
Download and process a tax update as a queued JSRPC job
my $param = thaw(decode_base64(shift));
my $format = $param->{'format'}; #well... this is all cch specific
- my ( $count, $last, $min_sec, $imported ) = (0, time, 5, 0); #progressbar
- $count = 100;
+ my ( $imported, $last, $min_sec ) = _progressbar_foo();
if ( $job ) { # progress bar
- my $error = $job->update_statustext( int( 100 * $imported / $count ) );
+ my $error = $job->update_statustext( 0);
die $error if $error;
}
- my $dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc. '/taxdata';
+ my $cache_dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc. '/';
+ my $dir = $cache_dir. 'taxdata';
unless (-d $dir) {
mkdir $dir or die "can't create $dir: $!\n";
}
if ($format eq 'cch') {
- eval "use Text::CSV_XS;";
- die $@ if $@;
-
- eval "use XBase;";
- die $@ if $@;
+ my @namelist = qw( code detail geocode plus4 txmatrix zip );
my $conf = new FS::Conf;
die "direct download of tax data not enabled\n"
$dir .= '/cch';
- my $oldAutoCommit = $FS::UID::AutoCommit;
- local $FS::UID::AutoCommit = 0;
my $dbh = dbh;
my $error = '';
# really should get a table EXCLUSIVE lock here
# check if initial import or update
+ #
+ # relying on mkdir "$dir.new" as a mutex
my $sql = "SELECT count(*) from tax_rate WHERE data_vendor='$format'";
my $sth = $dbh->prepare($sql) or die $dbh->errstr;
$sth->execute() or die $sth->errstr;
- my $upgrade = $sth->fetchrow_arrayref->[0];
+ my $update = $sth->fetchrow_arrayref->[0];
# create cache and/or rotate old tax data
if (-d $dir) {
- if (-d "$dir.4") {
- opendir(my $dirh, "$dir.4") or die "failed to open $dir.4: $!\n";
+ if (-d "$dir.9") {
+ opendir(my $dirh, "$dir.9") or die "failed to open $dir.9: $!\n";
foreach my $file (readdir($dirh)) {
- unlink "$dir.4/$file" if (-f "$dir.4/$file");
+ unlink "$dir.9/$file" if (-f "$dir.9/$file");
}
closedir($dirh);
- rmdir "$dir.4";
+ rmdir "$dir.9";
}
- for (3, 2, 1) {
+ for (8, 7, 6, 5, 4, 3, 2, 1) {
if ( -e "$dir.$_" ) {
rename "$dir.$_", "$dir.". ($_+1) or die "can't rename $dir.$_: $!\n";
}
} else {
- die "can't find previous tax data\n" if $upgrade;
+ die "can't find previous tax data\n" if $update;
}
# fetch and unpack the zip files
- my $ua = new LWP::UserAgent;
- foreach my $url (split ',', $urls) {
- my @name = split '/', $url; #somewhat restrictive
- my $name = pop @name;
- $name =~ /(.*)/; # untaint that which we trust;
- $name = $1;
-
- open my $taxfh, ">$dir.new/$name" or die "Can't open $dir.new/$name: $!\n";
-
- my $res = $ua->request(
- new HTTP::Request( GET => $url),
- sub { #my ($data, $response_object) = @_;
- print $taxfh $_[0] or die "Can't write to $dir.new/$name: $!\n";
- my $content_length = $_[1]->content_length;
- $imported += length($_[0]);
- if ( time - $min_sec > $last ) {
- my $error = $job->update_statustext(
- ($content_length ? int(100 * $imported/$content_length) : 0 ).
- ",Downloading data from CCH"
- );
- die $error if $error;
- $last = time;
- }
- },
- );
- die "download of $url failed: ". $res->status_line
- unless $res->is_success;
-
- close $taxfh;
- my $error = $job->update_statustext( "0,Unpacking data" );
- die $error if $error;
- $secret =~ /(.*)/; # untaint that which we trust;
- $secret = $1;
- system('unzip', "-P", $secret, "-d", "$dir.new", "$dir.new/$name") == 0
- or die "unzip -P $secret -d $dir.new $dir.new/$name failed";
- #unlink "$dir.new/$name";
- }
+ _cch_fetch_and_unzip( $job, $urls, $secret, "$dir.new" );
# extract csv files from the dbf files
- foreach my $name ( qw( code detail geocode plus4 txmatrix zip ) ) {
- my $error = $job->update_statustext( "0,Unpacking $name" );
- die $error if $error;
- warn "opening $dir.new/$name.dbf\n" if $DEBUG;
- my $table = new XBase 'name' => "$dir.new/$name.dbf";
- die "failed to access $dir.new/$name.dbf: ". XBase->errstr
- unless defined($table);
- $count = $table->last_record; # approximately;
- $imported = 0;
- open my $csvfh, ">$dir.new/$name.txt"
- or die "failed to open $dir.new/$name.txt: $!\n";
-
- my $csv = new Text::CSV_XS { 'always_quote' => 1 };
- my @fields = $table->field_names;
- my $cursor = $table->prepare_select;
- my $format_date =
- sub { my $date = shift;
- $date =~ /^(\d{4})(\d{2})(\d{2})$/ && ($date = "$2/$3/$1");
- $date;
- };
- while (my $row = $cursor->fetch_hashref) {
- $csv->combine( map { ($table->field_type($_) eq 'D')
- ? &{$format_date}($row->{$_})
- : $row->{$_}
- }
- @fields
- );
- print $csvfh $csv->string, "\n";
- $imported++;
- if ( time - $min_sec > $last ) {
- my $error = $job->update_statustext(
- int(100 * $imported/$count). ",Unpacking $name"
- );
- die $error if $error;
- $last = time;
- }
- }
- $table->close;
- close $csvfh;
+ foreach my $name ( @namelist ) {
+ _cch_extract_csv_from_dbf( $job, $dir, $name );
}
# generate the diff files
- my @insert_list = ();
- my @delete_list = ();
- my @predelete_list = ();
-
- my @list = (
- 'geocode', \&FS::tax_rate_location::batch_import,
- 'code', \&FS::tax_class::batch_import,
- 'plus4', \&FS::cust_tax_location::batch_import,
- 'zip', \&FS::cust_tax_location::batch_import,
- 'txmatrix', \&FS::part_pkg_taxrate::batch_import,
- 'detail', \&FS::tax_rate::batch_import,
- );
-
- while( scalar(@list) ) {
- my ( $name, $method ) = ( shift @list, shift @list );
- my %oldlines = ();
-
- my $error = $job->update_statustext( "0,Comparing to previous $name" );
- die $error if $error;
-
- warn "processing $dir.new/$name.txt\n" if $DEBUG;
-
- if ($upgrade) {
- open my $oldcsvfh, "$dir.1/$name.txt"
- or die "failed to open $dir.1/$name.txt: $!\n";
-
- while(<$oldcsvfh>) {
- chomp;
- $oldlines{$_} = 1;
- }
- close $oldcsvfh;
- }
-
- open my $newcsvfh, "$dir.new/$name.txt"
- or die "failed to open $dir.new/$name.txt: $!\n";
-
- my $ifh = new File::Temp( TEMPLATE => "$name.insert.XXXXXXXX",
- DIR => "$dir.new",
- UNLINK => 0, #meh
- ) or die "can't open temp file: $!\n";
-
- my $dfh = new File::Temp( TEMPLATE => "$name.delete.XXXXXXXX",
- DIR => "$dir.new",
- UNLINK => 0, #meh
- ) or die "can't open temp file: $!\n";
-
- while(<$newcsvfh>) {
- chomp;
- if (exists($oldlines{$_})) {
- $oldlines{$_} = 0;
- } else {
- print $ifh $_, ',"I"', "\n";
- }
- }
- close $newcsvfh;
-
- if ($name eq 'detail') {
- for (keys %oldlines) { # one file for rate details
- print $ifh $_, ',"D"', "\n" if $oldlines{$_};
- }
- } else {
- for (keys %oldlines) {
- print $dfh $_, ',"D"', "\n" if $oldlines{$_};
- }
- }
- %oldlines = ();
-
- push @insert_list, $name, $ifh->filename, $method;
- if ( $name eq 'geocode' ) {
- unshift @predelete_list, $name, $dfh->filename, $method
- unless $name eq 'detail';
- } else {
- unshift @delete_list, $name, $dfh->filename, $method
- unless $name eq 'detail';
+ my @list = ();
+ foreach my $name ( @namelist ) {
+ my $difffile = "$dir.new/$name.txt";
+ if ($update) {
+ my $error = $job->update_statustext( "0,Comparing to previous $name" );
+ die $error if $error;
+ warn "processing $dir.new/$name.txt\n" if $DEBUG;
+ my $olddir = $update ? "$dir.1" : "";
+ $difffile = _perform_cch_diff( $name, "$dir.new", $olddir );
}
-
- close $dfh;
- close $ifh;
+ $difffile =~ s/^$cache_dir//;
+ push @list, "${name}file:$difffile";
}
- while( scalar(@predelete_list) ) {
- my ($name, $file, $method) =
- (shift @predelete_list, shift @predelete_list, shift @predelete_list);
-
- my $fmt = "$format-update";
- $fmt = $fmt. ( $name eq 'zip' ? '-zip' : '' );
- open my $fh, "< $file" or $error ||= "Can't open $name file $file: $!";
- $error ||=
- &{$method}({ 'filehandle' => $fh, 'format' => $fmt }, $job);
- close $fh;
- #unlink $file or warn "Can't delete $file: $!";
- }
-
- while( scalar(@insert_list) ) {
- my ($name, $file, $method) =
- (shift @insert_list, shift @insert_list, shift @insert_list);
-
- my $fmt = "$format-update";
- $fmt = $fmt. ( $name eq 'zip' ? '-zip' : '' );
- open my $fh, "< $file" or $error ||= "Can't open $name file $file: $!";
- $error ||=
- &{$method}({ 'filehandle' => $fh, 'format' => $fmt }, $job);
- close $fh;
- #unlink $file or warn "Can't delete $file: $!";
- }
-
- while( scalar(@delete_list) ) {
- my ($name, $file, $method) =
- (shift @delete_list, shift @delete_list, shift @delete_list);
-
- my $fmt = "$format-update";
- $fmt = $fmt. ( $name eq 'zip' ? '-zip' : '' );
- open my $fh, "< $file" or $error ||= "Can't open $name file $file: $!";
- $error ||=
- &{$method}({ 'filehandle' => $fh, 'format' => $fmt }, $job);
- close $fh;
- #unlink $file or warn "Can't delete $file: $!";
- }
+ # perform the import
+ local $keep_cch_files = 1;
+ $param->{uploaded_files} = join( ',', @list );
+ $param->{format} .= '-update' if $update;
+ $error ||=
+ _perform_batch_import( $job, encode_base64( nfreeze( $param ) ) );
- if ($error) {
- $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
- die $error;
- }else{
- $dbh->commit or die $dbh->errstr if $oldAutoCommit;
- }
-
rename "$dir.new", "$dir"
or die "cch tax update processed, but can't rename $dir.new: $!\n";
return ($query, "SELECT COUNT(*) FROM tax_rate $extra_sql");
}
-# _upgrade_data
-#
-# Used by FS::Upgrade to migrate to a new database.
-#
-#
+=item queue_liability_report PARAMS
-sub _upgrade_data { # class method
- my ($self, %opts) = @_;
- my $dbh = dbh;
+Launches a tax liability report.
+
+PARAMS needs to be a base64-encoded Storable hash containing:
+- beginning: the start date, as a I<user-readable string> (not a timestamp).
+- end: the end date of the report, likewise.
+- agentnum: the agent to limit the report to, if any.
+
+=cut
+
+sub queue_liability_report {
+ my $job = shift;
+ my $param = thaw(decode_base64(shift));
- warn "$me upgrading $self\n" if $DEBUG;
+ my $cgi = new CGI;
+ $cgi->param('beginning', $param->{beginning});
+ $cgi->param('ending', $param->{ending});
+ my($beginning, $ending) = FS::UI::Web::parse_beginning_ending($cgi);
+ my $agentnum = $param->{agentnum};
+ if ($agentnum =~ /^(\d+)$/) { $agentnum = $1; } else { $agentnum = ''; };
+ generate_liability_report(
+ 'beginning' => $beginning,
+ 'ending' => $ending,
+ 'agentnum' => $agentnum,
+ 'p' => $param->{RootURL},
+ 'job' => $job,
+ );
+}
- my @column = qw ( tax excessrate usetax useexcessrate fee excessfee
- feebase feemax );
+=item generate_liability_report PARAMS
- if ( $dbh->{Driver}->{Name} eq 'Pg' ) {
+Generates a tax liability report. PARAMS must include:
- eval "use DBI::Const::GetInfoType;";
- die $@ if $@;
+- beginning, as a timestamp
+- ending, as a timestamp
+- p: the Freeside root URL, for generating links
+- agentnum (optional)
- my $major_version = 0;
- $dbh->get_info( $GetInfoType{SQL_DBMS_VER} ) =~ /^(\d{2})/
- && ( $major_version = sprintf("%d", $1) );
+=cut
- if ( $major_version > 7 ) {
+#shit, all sorts of false laxiness w/report_newtax.cgi
+sub generate_liability_report {
+ my %args = @_;
- # ideally this would be supported in DBIx-DBSchema and friends
+ my ( $count, $last, $min_sec ) = _progressbar_foo();
- foreach my $column ( @column ) {
- my $columndef = dbdef->table($self->table)->column($column);
- unless ($columndef->type eq 'numeric') {
+ #let us open the temp file early
+ my $dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc;
+ my $report = new File::Temp( TEMPLATE => 'report.tax.liability.XXXXXXXX',
+ DIR => $dir,
+ UNLINK => 0, # not so temp
+ ) or die "can't open report file: $!\n";
- warn "updating tax_rate column $column to numeric\n" if $DEBUG;
- my $sql = "ALTER TABLE tax_rate ALTER $column TYPE numeric(14,8)";
- my $sth = $dbh->prepare($sql) or die $dbh->errstr;
- $sth->execute or die $sth->errstr;
+ my $conf = new FS::Conf;
+ my $money_char = $conf->config('money_char') || '$';
- warn "updating h_tax_rate column $column to numeric\n" if $DEBUG;
- $sql = "ALTER TABLE h_tax_rate ALTER $column TYPE numeric(14,8)";
- $sth = $dbh->prepare($sql) or die $dbh->errstr;
- $sth->execute or die $sth->errstr;
+ my $join_cust = "
+ JOIN cust_bill USING ( invnum )
+ LEFT JOIN cust_main USING ( custnum )
+ ";
- }
+ my $join_loc =
+ "LEFT JOIN cust_bill_pkg_tax_rate_location USING ( billpkgnum )";
+ my $join_tax_loc = "LEFT JOIN tax_rate_location USING ( taxratelocationnum )";
+
+ my $addl_from = " $join_cust $join_loc $join_tax_loc ";
+
+ my $where = "WHERE _date >= $args{beginning} AND _date <= $args{ending} ";
+
+ my $agentname = '';
+ if ( $args{agentnum} =~ /^(\d+)$/ ) {
+ my $agent = qsearchs('agent', { 'agentnum' => $1 } );
+ die "agent not found" unless $agent;
+ $agentname = $agent->agent;
+ $where .= ' AND cust_main.agentnum = '. $agent->agentnum;
+ }
+
+ #my @taxparam = ( 'itemdesc', 'tax_rate_location.state', 'tax_rate_location.county', 'tax_rate_location.city', 'cust_bill_pkg_tax_rate_location.locationtaxid' );
+ my @taxparams = qw( city county state locationtaxid );
+ my @params = ('itemdesc', @taxparams);
+
+ my $select = 'DISTINCT itemdesc,locationtaxid,tax_rate_location.state,tax_rate_location.county,tax_rate_location.city';
+
+ #false laziness w/FS::Report::Table::Monthly (sub should probably be moved up
+ #to FS::Report or FS::Record or who the fuck knows where)
+ my $scalar_sql = sub {
+ my( $r, $param, $sql ) = @_;
+ my $sth = dbh->prepare($sql) or die dbh->errstr;
+ $sth->execute( map $r->$_(), @$param )
+ or die "Unexpected error executing statement $sql: ". $sth->errstr;
+ $sth->fetchrow_arrayref->[0] || 0;
+ };
+
+ my $tax = 0;
+ my $credit = 0;
+ my %taxes = ();
+ my %basetaxes = ();
+ my $calculated = 0;
+
+ # get all distinct tuples of (tax name, state, county, city, locationtaxid)
+ # for taxes that have been charged
+ # (state, county, city are from tax_rate_location, not from customer data)
+ my @tax_and_location = qsearch({ table => 'cust_bill_pkg',
+ select => $select,
+ hashref => { pkgpart => 0 },
+ addl_from => $addl_from,
+ extra_sql => $where,
+ debug => 1,
+ });
+ $count = scalar(@tax_and_location);
+ foreach my $t ( @tax_and_location ) {
+
+ if ( $args{job} ) {
+ if ( time - $min_sec > $last ) {
+ $args{job}->update_statustext( int( 100 * $calculated / $count ).
+ ",Calculating"
+ );
+ $last = time;
}
+ }
- } elsif ( $dbh->{pg_server_version} =~ /^704/ ) {
+ #my @params = map { my $f = $_; $f =~ s/.*\.//; $f } @taxparam;
+ my $label = join('~', map { $t->$_ } @params);
+ $label = 'Tax'. $label if $label =~ /^~/;
+ unless ( exists( $taxes{$label} ) ) {
+ my ($baselabel, @trash) = split /~/, $label;
- # ideally this would be supported in DBIx-DBSchema and friends
+ $taxes{$label}->{'label'} = join(', ', split(/~/, $label) );
+ $taxes{$label}->{'url_param'} =
+ join(';', map { "$_=". uri_escape($t->$_) } @params);
- foreach my $column ( @column ) {
- my $columndef = dbdef->table($self->table)->column($column);
- unless ($columndef->type eq 'numeric') {
+ my $itemdesc_loc =
+ # " payby != 'COMP' ". # breaks the entire report under 4.x
+ # # and unnecessary since COMP accounts don't
+ # # get taxes calculated in the first place
+ " ( itemdesc = ? OR ? = '' AND itemdesc IS NULL ) ".
+ "AND ". FS::tax_rate_location->location_sql( map { $_ => $t->$_ }
+ @taxparams
+ );
- warn "updating tax_rate column $column to numeric\n" if $DEBUG;
+ my $taxwhere =
+ "FROM cust_bill_pkg $addl_from $where AND $itemdesc_loc";
- foreach my $table ( qw( tax_rate h_tax_rate ) ) {
+ my $sql = "SELECT SUM(amount) $taxwhere AND cust_bill_pkg.pkgnum = 0";
- my $sql = "ALTER TABLE $table RENAME $column TO old_$column";
- my $sth = $dbh->prepare($sql) or die $dbh->errstr;
- $sth->execute or die $sth->errstr;
+ my $x = &{$scalar_sql}($t, [ 'itemdesc', 'itemdesc' ], $sql );
+ $tax += $x;
+ $taxes{$label}->{'tax'} += $x;
- my $def = dbdef->table($table)->column($column);
- $def->type('numeric');
- $def->length('14,8');
- my $null = $def->null;
- $def->null('NULL');
+ my $creditfrom =
+ "JOIN cust_credit_bill_pkg USING (billpkgnum,billpkgtaxratelocationnum)";
+ my $creditwhere =
+ "FROM cust_bill_pkg $addl_from $creditfrom $where AND $itemdesc_loc";
- $sql = "ALTER TABLE $table ADD COLUMN ". $def->line($dbh);
- $sth = $dbh->prepare($sql) or die $dbh->errstr;
- $sth->execute or die $sth->errstr;
+ $sql = "SELECT SUM(cust_credit_bill_pkg.amount) ".
+ " $creditwhere AND cust_bill_pkg.pkgnum = 0";
- $sql = "UPDATE $table SET $column = CAST( old_$column AS numeric )";
- $sth = $dbh->prepare($sql) or die $dbh->errstr;
- $sth->execute or die $sth->errstr;
+ my $y = &{$scalar_sql}($t, [ 'itemdesc', 'itemdesc' ], $sql );
+ $credit += $y;
+ $taxes{$label}->{'credit'} += $y;
- unless ( $null eq 'NULL' ) {
- $sql = "ALTER TABLE $table ALTER $column SET NOT NULL";
- $sth = $dbh->prepare($sql) or die $dbh->errstr;
- $sth->execute or die $sth->errstr;
- }
+ unless ( exists( $taxes{$baselabel} ) ) {
- $sql = "ALTER TABLE $table DROP old_$column";
- $sth = $dbh->prepare($sql) or die $dbh->errstr;
- $sth->execute or die $sth->errstr;
+ $basetaxes{$baselabel}->{'label'} = $baselabel;
+ $basetaxes{$baselabel}->{'url_param'} = "itemdesc=$baselabel";
+ $basetaxes{$baselabel}->{'base'} = 1;
- }
- }
}
- } else {
+ $basetaxes{$baselabel}->{'tax'} += $x;
+ $basetaxes{$baselabel}->{'credit'} += $y;
+
+ }
- warn "WARNING: tax_rate table upgrade unsupported for this Pg version\n";
+ # calculate customer-exemption for this tax
+ # calculate package-exemption for this tax
+ # calculate monthly exemption (texas tax) for this tax
+ # count up all the cust_tax_exempt_pkg records associated with
+ # the actual line items.
+ }
- }
- } else {
+ #ordering
- warn "WARNING: tax_rate table upgrade only supported for Pg 8+\n";
+ if ( $args{job} ) {
+ $args{job}->update_statustext( "0,Sorted" );
+ $last = time;
+ }
+ my @taxes = ();
+
+ foreach my $tax ( sort { $a cmp $b } keys %taxes ) {
+ my ($base, @trash) = split '~', $tax;
+ my $basetax = delete( $basetaxes{$base} );
+ if ($basetax) {
+ if ( $basetax->{tax} == $taxes{$tax}->{tax} ) {
+ $taxes{$tax}->{base} = 1;
+ } else {
+ push @taxes, $basetax;
+ }
+ }
+ push @taxes, $taxes{$tax};
}
- '';
+ push @taxes, {
+ 'label' => 'Total',
+ 'url_param' => '',
+ 'tax' => $tax,
+ 'credit' => $credit,
+ 'base' => 1,
+ };
+
+
+ my $dateagentlink = "begin=$args{beginning};end=$args{ending}";
+ $dateagentlink .= ';agentnum='. $args{agentnum}
+ if length($agentname);
+ my $baselink = $args{p}. "search/cust_bill_pkg.cgi?vendortax=1;" .
+ $dateagentlink;
+ my $creditlink = $args{p}. "search/cust_credit_bill_pkg.html?$dateagentlink";
+
+ print $report <<EOF;
+
+ <% include("/elements/header.html", "$agentname Tax Report - ".
+ ( $args{beginning}
+ ? time2str('%h %o %Y ', $args{beginning} )
+ : ''
+ ).
+ 'through '.
+ ( $args{ending} == 4294967295
+ ? 'now'
+ : time2str('%h %o %Y', $args{ending} )
+ )
+ )
+ %>
+
+ <% include('/elements/table-grid.html') %>
+
+ <TR>
+ <TH CLASS="grid" BGCOLOR="#cccccc"></TH>
+ <TH CLASS="grid" BGCOLOR="#cccccc"></TH>
+ <TH CLASS="grid" BGCOLOR="#cccccc">Tax invoiced</TH>
+ <TH CLASS="grid" BGCOLOR="#cccccc"> </TH>
+ <TH CLASS="grid" BGCOLOR="#cccccc"></TH>
+ <TH CLASS="grid" BGCOLOR="#cccccc">Tax credited</TH>
+ </TR>
+EOF
+
+ my $bgcolor1 = '#eeeeee';
+ my $bgcolor2 = '#ffffff';
+ my $bgcolor = '';
+
+ $count = scalar(@taxes);
+ $calculated = 0;
+ foreach my $tax ( @taxes ) {
+
+ if ( $args{job} ) {
+ if ( time - $min_sec > $last ) {
+ $args{job}->update_statustext( int( 100 * $calculated / $count ).
+ ",Generated"
+ );
+ $last = time;
+ }
+ }
+
+ if ( $bgcolor eq $bgcolor1 ) {
+ $bgcolor = $bgcolor2;
+ } else {
+ $bgcolor = $bgcolor1;
+ }
+
+ my $link = '';
+ if ( $tax->{'label'} ne 'Total' ) {
+ $link = ';'. $tax->{'url_param'};
+ }
+
+ print $report <<EOF;
+ <TR>
+ <TD CLASS="grid" BGCOLOR="<% '$bgcolor' %>"><% '$tax->{label}' %></TD>
+ <% ($tax->{base}) ? qq!<TD CLASS="grid" BGCOLOR="$bgcolor"></TD>! : '' %>
+ <TD CLASS="grid" BGCOLOR="<% '$bgcolor' %>" ALIGN="right">
+ <A HREF="<% '$baselink$link' %>;istax=1"><% '$money_char' %><% sprintf('%.2f', $tax->{'tax'} ) %></A>
+ </TD>
+ <% !($tax->{base}) ? qq!<TD CLASS="grid" BGCOLOR="$bgcolor"></TD>! : '' %>
+ <TD CLASS="grid" BGCOLOR="<% '$bgcolor' %>"></TD>
+ <% ($tax->{base}) ? qq!<TD CLASS="grid" BGCOLOR="$bgcolor"></TD>! : '' %>
+ <TD CLASS="grid" BGCOLOR="<% '$bgcolor' %>" ALIGN="right">
+ <A HREF="<% '$creditlink$link' %>;istax=1;iscredit=rate"><% '$money_char' %><% sprintf('%.2f', $tax->{'credit'} ) %></A>
+ </TD>
+ <% !($tax->{base}) ? qq!<TD CLASS="grid" BGCOLOR="$bgcolor"></TD>! : '' %>
+ </TR>
+EOF
+ }
+
+ print $report <<EOF;
+ </TABLE>
+
+ </BODY>
+ </HTML>
+EOF
+
+ my $reportname = $report->filename;
+ close $report;
+
+ my $dropstring = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc. '/report.';
+ $reportname =~ s/^$dropstring//;
+
+ my $reporturl = "%%%ROOTURL%%%/misc/queued_report?report=$reportname";
+ die "<a href=$reporturl>view</a>\n";
}
+
+
=back
=head1 BUGS
Mixing automatic and manual editing works poorly at present.
+ Tax liability calculations take too long and arguably don't belong here.
+ Tax liability report generation not entirely safe (escaped).
+
=head1 SEE ALSO
-L<FS::Record>, L<FS::cust_main>, L<FS::cust_bill>, schema.html from the base
-documentation.
+L<FS::Record>, L<FS::cust_location>, L<FS::cust_bill>
=cut