X-Git-Url: http://git.freeside.biz/gitweb/?p=freeside.git;a=blobdiff_plain;f=FS%2FFS%2Ftax_rate.pm;h=c6fe243d4a500e0978324edb50bd20191bbef772;hp=93550b178c5d20b53f8d274a8ba7eb2e44c5aebb;hb=a36e0f8a0f69349dafaa16d1d2d57dfb6e5dbc85;hpb=250e277720fbe288875736c7f5f957668f4c1880 diff --git a/FS/FS/tax_rate.pm b/FS/FS/tax_rate.pm index 93550b178..c6fe243d4 100644 --- a/FS/FS/tax_rate.pm +++ b/FS/FS/tax_rate.pm @@ -1,23 +1,26 @@ package FS::tax_rate; +use base qw( FS::Record ); use strict; -use vars qw( @ISA $DEBUG $me +use vars qw( $DEBUG $me %tax_unittypes %tax_maxtypes %tax_basetypes %tax_authorities - %tax_passtypes %GetInfoType ); + %tax_passtypes %GetInfoType $keep_cch_files ); use Date::Parse; use DateTime; use DateTime::Format::Strptime; -use Storable qw( thaw ); use IO::File; use File::Temp; +use Text::CSV_XS; +use URI::Escape; use LWP::UserAgent; use HTTP::Request; use HTTP::Response; -use MIME::Base64; use DBIx::DBSchema; use DBIx::DBSchema::Table; use DBIx::DBSchema::Column; +use List::Util 'sum'; use FS::Record qw( qsearch qsearchs dbh dbdef ); +use FS::Conf; use FS::tax_class; use FS::cust_bill_pkg; use FS::cust_tax_location; @@ -27,10 +30,9 @@ use FS::part_pkg_taxproduct; use FS::cust_main; use FS::Misc qw( csv_from_fixed ); -@ISA = qw( FS::Record ); - $DEBUG = 0; $me = '[FS::tax_rate]'; +$keep_cch_files = 0; =head1 NAME @@ -77,9 +79,10 @@ a location code provided by a tax authority =item taxclassnum -a foreign key into FS::tax_class - the type of tax -referenced but FS::part_pkg_taxrate -eitem effective_date +a foreign key into FS::tax_class - the type of tax referenced by +FS::part_pkg_taxrate + +=item effective_date the time after which the tax applies @@ -210,8 +213,8 @@ sub check { $self->ut_numbern('taxnum') || $self->ut_text('geocode') || $self->ut_textn('data_vendor') - || $self->ut_textn('location') - || $self->ut_foreign_key('taxclassnum', 'tax_class', 'taxclassnum') + || $self->ut_cch_textn('location') + || $self->ut_foreign_keyn('taxclassnum', 'tax_class', 'taxclassnum') || $self->ut_snumbern('effective_date') || $self->ut_float('tax') || $self->ut_floatn('excessrate') @@ -240,6 +243,18 @@ sub check { } +#ut_text / ut_textn w/ ` added cause now that's in the data +sub ut_cch_textn { + my($self,$field)=@_; + $self->getfield($field) + =~ /^([\wô \!\@\#\$\%\&\(\)\-\+\;\:\'\"\,\.\?\/\=\[\]\<\>\`]*)$/ + or return gettext('illegal_or_empty_text'). " $field: ". + $self->getfield($field); + $self->setfield($field,$1); + ''; + +} + =item taxclass_description Returns the human understandable value associated with the related @@ -271,16 +286,25 @@ sub unittype_name { =item maxtype_name -Returns the human understandable value associated with the maxtype column +Returns the human understandable value associated with the maxtype column. =cut +# XXX these are non-functional, and most of them are horrible to implement +# in our current model + %tax_maxtypes = ( '0' => 'receipts per invoice', '1' => 'receipts per item', '2' => 'total utility charges per utility tax year', '3' => 'total charges per utility tax year', '4' => 'receipts per access line', + '7' => 'total utility charges per calendar year', '9' => 'monthly receipts per location', + '10' => 'monthly receipts exceeds taxbase and total tax per month does not exceed maxtax', # wtf? + '11' => 'receipts/units per access line', + '14' => 'units per invoice', + '15' => 'units per month', + '18' => 'units per account', ); sub maxtype_name { @@ -356,68 +380,69 @@ sub passtype_name { $tax_passtypes{$self->passtype}; } -=item taxline TAXABLES, [ OPTIONSHASH ] +=item taxline_cch TAXABLES, CLASSES + +Takes an arrayref of L objects representing taxable line +items, and an arrayref of charge classes ('setup', 'recur', '' for +unclassified usage, or an L number). Calculates the tax on +each item under this tax definition and returns a list of new +L objects for the taxes charged. -Returns a listref of a name and an amount of tax calculated for the list -of packages/amounts referenced by TAXABLES. If an error occurs, a message -is returned as a scalar. +If the taxable objects are linked to an invoice, this will also calculate +per-customer exemptions (cust_exempt and cust_taxname_exempt) and attach them +to the line items in the 'cust_tax_exempt_pkg' pseudo-field. + +For accurate calculation of per-customer or per-location taxes, ALL items +appearing on the invoice (and subject to this tax) MUST be passed to this +method together, and NO items from any other invoice should be included. =cut -sub taxline { +sub taxline_cch { my $self = shift; + # this used to accept a hash of options but none of them did anything + # so it's been removed. - my $taxables; - my %opt = (); - - if (ref($_[0]) eq 'ARRAY') { - $taxables = shift; - %opt = @_; - }else{ - $taxables = [ @_ ]; - #exemptions would be broken in this case - } + my $taxables = shift; + my $classes = shift || []; my $name = $self->taxname; $name = 'Other surcharges' if ($self->passtype == 2); my $amount = 0; - - if ( $self->disabled ) { # we always know how to handle disabled taxes - return { - 'name' => $name, - 'amount' => $amount, - }; - } + + return unless @$taxables; # nothing to do + return if $self->disabled; + return if $self->passflag eq 'N'; # tax can't be passed to the customer + # but should probably still appear on the liability report--create a + # cust_tax_exempt_pkg record for it? + + # in 4.x, the invoice is _already inserted_ before we try to calculate + # tax on it. though it may be a quotation, so be careful. + + my $cust_main; + my $cust_bill = $taxables->[0]->cust_bill; + $cust_main = $cust_bill->cust_main if $cust_bill; my $taxable_charged = 0; my @cust_bill_pkg = grep { $taxable_charged += $_ unless ref; ref; } @$taxables; + my $taxratelocationnum = $self->tax_rate_location->taxratelocationnum; + warn "calculating taxes for ". $self->taxnum. " on ". join (",", map { $_->pkgnum } @cust_bill_pkg) if $DEBUG; - if ($self->passflag eq 'N') { - # return "fatal: can't (yet) handle taxes not passed to the customer"; - # until someone needs to track these in freeside - return { - 'name' => $name, - 'amount' => 0, - }; - } - - if ($self->maxtype != 0 && $self->maxtype != 9) { + my $maxtype = $self->maxtype || 0; + if ($maxtype != 0 && $maxtype != 1 + && $maxtype != 14 && $maxtype != 15 + && $maxtype != 18 # sigh + ) { return $self->_fatal_or_null( 'tax with "'. $self->maxtype_name. '" threshold' ); - } - - if ($self->maxtype == 9) { - return - $self->_fatal_or_null( 'tax with "'. $self->maxtype_name. '" threshold' ); - # "texas" tax - } + } # I don't know why, it's not like there are maxtypes that we DO support # we treat gross revenue as gross receipts and expect the tax data # to DTRT (i.e. tax on tax rules) @@ -430,85 +455,211 @@ sub taxline { $self->_fatal_or_null( 'tax with "'. $self->basetype_name. '" basis' ); } - unless ($self->setuptax =~ /^Y$/i) { - $taxable_charged += $_->setup foreach @cust_bill_pkg; - } - unless ($self->recurtax =~ /^Y$/i) { - $taxable_charged += $_->recur foreach @cust_bill_pkg; - } + my @tax_links; # for output + my %seen; # locationnum or pkgnum => 1 + my $taxable_cents = 0; my $taxable_units = 0; - unless ($self->recurtax =~ /^Y$/i) { - if ($self->unittype == 0) { - my %seen = (); - foreach (@cust_bill_pkg) { - $taxable_units += $_->units - unless $seen{$_->pkgnum}; - $seen{$_->pkgnum}++; + my $tax_cents = 0; + + while (@$taxables) { + my $cust_bill_pkg = shift @$taxables; + my $class = shift @$classes; + $class = 'all' if !defined($class); + + my %usage_map = map { $_ => $cust_bill_pkg->usage($_) } + $cust_bill_pkg->usage_classes; + my $usage_total = sum( values(%usage_map), 0 ); + + # determine if the item has exemptions that apply to this tax def + my @exemptions = grep { $_->taxnum == $self->taxnum } + @{ $cust_bill_pkg->cust_tax_exempt_pkg }; + + if ( $self->tax > 0 ) { + + my $taxable_charged = 0; + if ($class eq 'all') { + $taxable_charged = $cust_bill_pkg->setup + $cust_bill_pkg->recur; + } elsif ($class eq 'setup') { + $taxable_charged = $cust_bill_pkg->setup; + } elsif ($class eq 'recur') { + $taxable_charged = $cust_bill_pkg->recur - $usage_total; + } else { + $taxable_charged = $usage_map{$class} || 0; } - }elsif ($self->unittype == 1) { - return $self->_fatal_or_null( 'fee with minute unit type' ); - }elsif ($self->unittype == 2) { - $taxable_units = 1; - }else { - return $self->_fatal_or_null( 'unknown unit type in tax'. $self->taxnum ); - } - } - # - # XXX insert exemption handling here - # - # the tax or fee is applied to taxbase or feebase and then - # the excessrate or excess fee is applied to taxmax or feemax - # + foreach my $ex (@exemptions) { + # the only cases where the exemption doesn't apply: + # if it's a setup exemption and $class is not 'setup' or 'all' + # if it's a recur exemption and $class is 'setup' + if ( ( $ex->exempt_recur and $class eq 'setup' ) + or ( $ex->exempt_setup and $class ne 'setup' and $class ne 'all' ) + ) { + next; + } + + $taxable_charged -= $ex->amount; + } + # cust_main_county handles monthly capped exemptions; this doesn't. + # + # $taxable_charged can also be less than zero at this point + # (recur exemption + usage class breakdown); treat that as zero. + next if $taxable_charged <= 0; + + # yeah, some false laziness with cust_main_county + my $this_tax_cents = int(100 * $taxable_charged * $self->tax); + my $tax_link = FS::cust_bill_pkg_tax_rate_location->new({ + 'taxnum' => $self->taxnum, + 'taxtype' => ref($self), + 'cents' => $this_tax_cents, # not a real field + 'locationtaxid' => $self->location, # fundamentally silly + 'taxable_billpkgnum' => $cust_bill_pkg->billpkgnum, + 'taxable_cust_bill_pkg' => $cust_bill_pkg, + 'taxratelocationnum' => $taxratelocationnum, + 'taxclass' => $class, + }); + push @tax_links, $tax_link; + + $taxable_cents += 100 * $taxable_charged; + $tax_cents += $this_tax_cents; + + } elsif ( $self->fee > 0 ) { + # most CCH taxes are this type, because nearly every county has a 911 + # fee + my $units = 0; + + # since we don't support partial exemptions (except setup/recur), + # if there's an exemption that applies to this package and taxrate, + # don't charge ANY per-unit fees + next if @exemptions; + + # don't apply fees to usage classes (maybe if we ever get per-minute + # fees?) + next unless $class eq 'setup' + or $class eq 'recur' + or $class eq 'all'; + + if ( $self->unittype == 0 ) { + if ( !$seen{$cust_bill_pkg->pkgnum} ) { + # per access line + $units = $cust_bill_pkg->units; + $seen{$cust_bill_pkg->pkgnum} = 1; + } # else it's been seen, leave it at zero units + + } elsif ($self->unittype == 1) { # per minute + # STILL not supported...fortunately these only exist if you happen + # to be in Idaho or Little Rock, Arkansas + # + # though a voip_cdr package could easily report minutes of usage... + return $self->_fatal_or_null( 'fee with minute unit type' ); + + } elsif ( $self->unittype == 2 ) { + + # per account + my $locationnum = $cust_bill_pkg->tax_locationnum; + if (!$locationnum and $cust_main) { + $locationnum = $cust_main->ship_locationnum; + } + # the other case is that it's a quotation + + $units = 1 unless $seen{$cust_bill_pkg->tax_locationnum}; + $seen{$cust_bill_pkg->tax_locationnum} = 1; - $amount += $taxable_charged * $self->tax; - $amount += $taxable_units * $self->fee; - - warn "calculated taxes as [ $name, $amount ]\n" - if $DEBUG; + } else { + # Unittype 19 is used for prepaid wireless E911 charges in many states. + # Apparently "per retail purchase", which for us would mean per invoice. + # Unittype 20 is used for some 911 surcharges and I have no idea what + # it means. + return $self->_fatal_or_null( 'unknown unit type in tax'. $self->taxnum ); + } + my $this_tax_cents = int($units * $self->fee * 100); + my $tax_link = FS::cust_bill_pkg_tax_rate_location->new({ + 'taxnum' => $self->taxnum, + 'taxtype' => ref($self), + 'cents' => $this_tax_cents, + 'locationtaxid' => $self->location, + 'taxable_billpkgnum' => $cust_bill_pkg->billpkgnum, + 'taxable_cust_bill_pkg' => $cust_bill_pkg, + 'taxratelocationnum' => $taxratelocationnum, + 'taxclass' => $class, + }); + push @tax_links, $tax_link; + + $taxable_units += $units; + $tax_cents += $this_tax_cents; - return { - 'name' => $name, - 'amount' => $amount, - }; + } + } # foreach $cust_bill_pkg + + # check bracket maxima; throw an error if we've gone over, because + # we don't really implement them + + if ( ($self->taxmax > 0 and $taxable_cents > $self->taxmax*100 ) or + ($self->feemax > 0 and $taxable_units > $self->feemax) ) { + # throw an error + # (why not just cap taxable_charged/units at the taxmax/feemax? because + # it's way more complicated than that. this won't even catch every case + # where a bracket maximum should apply.) + return $self->_fatal_or_null( 'tax base > taxmax/feemax for tax'.$self->taxnum ); + } + + # round and distribute + my $total_tax_cents = sprintf('%.0f', + ($taxable_cents * $self->tax) + ($taxable_units * $self->fee * 100) + ); + my $extra_cents = sprintf('%.0f', $total_tax_cents - $tax_cents); + $tax_cents += $extra_cents; + my $i = 0; + foreach (@tax_links) { # can never require more than a single pass, yes? + my $cents = $_->get('cents'); + if ( $extra_cents > 0 ) { + $cents++; + $extra_cents--; + } + $_->set('amount', sprintf('%.2f', $cents/100)); + } + return @tax_links; } sub _fatal_or_null { my ($self, $error) = @_; + $DB::single = 1; # not a mistake + my $conf = new FS::Conf; - $error = "fatal: can't yet handle ". $error; + $error = "can't yet handle ". $error; my $name = $self->taxname; $name = 'Other surcharges' if ($self->passtype == 2); if ($conf->exists('ignore_incalculable_taxes')) { - warn $error; + warn "WARNING: $error; billing anyway per ignore_incalculable_taxes conf\n"; return { name => $name, amount => 0 }; } else { - return $error; + return "fatal: $error"; } } -=item tax_on_tax CUST_MAIN +=item tax_on_tax CUST_LOCATION Returns a list of taxes which are candidates for taxing taxes for the -given customer (see L) +given service location (see L) =cut + #hot sub tax_on_tax { + #akshun my $self = shift; - my $cust_main = shift; + my $cust_location = shift; warn "looking up taxes on tax ". $self->taxnum. " for customer ". - $cust_main->custnum + $cust_location->custnum if $DEBUG; - my $geocode = $cust_main->geocode($self->data_vendor); + my $geocode = $cust_location->geocode($self->data_vendor); # CCH oddness in m2m my $dbh = dbh; @@ -564,6 +715,36 @@ sub tax_rate_location { } + +=item find_or_insert + +Finds an existing tax definition matching the data_vendor, taxname, +taxclassnum, and geocode of this one, if one exists, and sets the contents of +this tax rate equal to that one (including its taxnum). If an existing +definition is not found, inserts this one. Returns an error string if +inserting a record failed. + +=cut + +sub find_or_insert { + my $self = shift; + # this doesn't uniquely identify CCH taxes (kinda goofy, I know) + die "find_or_insert is not compatible with CCH taxes\n" + if $self->data_vendor eq 'cch'; + + my @keys = (qw(data_vendor taxname taxclassnum geocode)); + my %hash = map { $_ => $self->get($_) } @keys; + my $existing = qsearchs('tax_rate', \%hash); + if ($existing) { + foreach ($self->fields) { + $self->set($_, $existing->get($_)); + } + return; + } else { + return $self->insert; + } +} + =back =head1 SUBROUTINES @@ -574,6 +755,10 @@ sub tax_rate_location { =cut +sub _progressbar_foo { + return (0, time, 5); +} + sub batch_import { my ($param, $job) = @_; @@ -602,7 +787,7 @@ sub batch_import { } my $line; - my ( $count, $last, $min_sec ) = (0, time, 5); #progressbar + my ( $count, $last, $min_sec ) = _progressbar_foo(); if ( $job || scalar(@column_callbacks) ) { my $error = csv_from_fixed(\$fh, \$count, \@column_lengths, \@column_callbacks); @@ -611,6 +796,7 @@ sub batch_import { $count *=2; if ( $format eq 'cch' || $format eq 'cch-update' ) { + #false laziness w/below (sub _perform_cch_diff) @fields = qw( geocode inoutcity inoutlocal tax location taxbase taxmax excessrate effective_date taxauth taxtype taxcat taxname usetax useexcessrate fee unittype feemax maxtype passflag @@ -628,6 +814,7 @@ sub batch_import { my $dt = $parser->parse_datetime( $hash->{'effective_date'} ); $hash->{'effective_date'} = $dt ? $dt->epoch : ''; + $hash->{$_} =~ s/\s//g foreach qw( inoutcity inoutlocal ) ; $hash->{$_} = sprintf("%.2f", $hash->{$_}) foreach qw( taxbase taxmax ); my $taxclassid = @@ -688,9 +875,6 @@ sub batch_import { die "unknown format $format"; } - eval "use Text::CSV_XS;"; - die $@ if $@; - my $csv = new Text::CSV_XS; my $imported = 0; @@ -731,9 +915,10 @@ sub batch_import { foreach my $field ( @fields ) { $tax_rate{$field} = shift @columns; } + if ( scalar( @columns ) ) { $dbh->rollback if $oldAutoCommit; - return "Unexpected trailing columns in line (wrong format?): $line"; + return "Unexpected trailing columns in line (wrong format?) importing tax_rate: $line"; } my $error = &{$hook}(\%tax_rate); @@ -758,7 +943,8 @@ sub batch_import { } - for (grep { !exists($delete{$_}) } keys %insert) { + my @replace = grep { exists($delete{$_}) } keys %insert; + for (@replace) { if ( $job ) { # progress bar if ( time - $min_sec > $last ) { my $error = $job->update_statustext( @@ -772,20 +958,35 @@ sub batch_import { } } - my $tax_rate = new FS::tax_rate( $insert{$_} ); - my $error = $tax_rate->insert; + my $old = qsearchs( 'tax_rate', $delete{$_} ); - if ( $error ) { - $dbh->rollback if $oldAutoCommit; - my $hashref = $insert{$_}; - $line = join(", ", map { "$_ => ". $hashref->{$_} } keys(%$hashref) ); - return "can't insert tax_rate for $line: $error"; + if ( $old ) { + + my $new = new FS::tax_rate({ $old->hash, %{$insert{$_}}, 'manual' => '' }); + $new->taxnum($old->taxnum); + my $error = $new->replace($old); + + if ( $error ) { + $dbh->rollback if $oldAutoCommit; + my $hashref = $insert{$_}; + $line = join(", ", map { "$_ => ". $hashref->{$_} } keys(%$hashref) ); + return "can't replace tax_rate for $line: $error"; + } + + $imported++; + + } else { + + $old = delete $delete{$_}; + warn "WARNING: can't find tax_rate to replace (inserting instead and continuing) for: ". + #join(" ", map { "$_ => ". $old->{$_} } @fields); + join(" ", map { "$_ => ". $old->{$_} } keys(%$old) ); } $imported++; } - for (grep { exists($delete{$_}) } keys %insert) { + for (grep { !exists($delete{$_}) } keys %insert) { if ( $job ) { # progress bar if ( time - $min_sec > $last ) { my $error = $job->update_statustext( @@ -799,27 +1000,17 @@ sub batch_import { } } - my $old = qsearchs( 'tax_rate', $delete{$_} ); - unless ($old) { - $dbh->rollback if $oldAutoCommit; - $old = $delete{$_}; - return "can't find tax_rate to replace for: ". - #join(" ", map { "$_ => ". $old->{$_} } @fields); - join(" ", map { "$_ => ". $old->{$_} } keys(%$old) ); - } - my $new = new FS::tax_rate({ $old->hash, %{$insert{$_}}, 'manual' => '' }); - $new->taxnum($old->taxnum); - my $error = $new->replace($old); + my $tax_rate = new FS::tax_rate( $insert{$_} ); + my $error = $tax_rate->insert; if ( $error ) { $dbh->rollback if $oldAutoCommit; my $hashref = $insert{$_}; $line = join(", ", map { "$_ => ". $hashref->{$_} } keys(%$hashref) ); - return "can't replace tax_rate for $line: $error"; + return "can't insert tax_rate for $line: $error"; } $imported++; - $imported++; } for (grep { !exists($insert{$_}) } keys %delete) { @@ -837,20 +1028,22 @@ sub batch_import { } my $tax_rate = qsearchs( 'tax_rate', $delete{$_} ); - unless ($tax_rate) { + if (!$tax_rate) { $dbh->rollback if $oldAutoCommit; $tax_rate = $delete{$_}; - return "can't find tax_rate to delete for: ". - #join(" ", map { "$_ => ". $tax_rate->{$_} } @fields); - join(" ", map { "$_ => ". $tax_rate->{$_} } keys(%$tax_rate) ); - } - my $error = $tax_rate->delete; + warn "WARNING: can't find tax_rate to delete for: ". + join(" ", map { "$_ => ". $tax_rate->{$_} } keys(%$tax_rate) ). + " (ignoring)\n"; + } else { + my $error = $tax_rate->delete; # XXX we really should not do this + # (it orphans CBPTRL records) - if ( $error ) { - $dbh->rollback if $oldAutoCommit; - my $hashref = $delete{$_}; - $line = join(", ", map { "$_ => ". $hashref->{$_} } keys(%$hashref) ); - return "can't delete tax_rate for $line: $error"; + if ( $error ) { + $dbh->rollback if $oldAutoCommit; + my $hashref = $delete{$_}; + $line = join(", ", map { "$_ => ". $hashref->{$_} } keys(%$hashref) ); + return "can't delete tax_rate for $line: $error"; + } } $imported++; @@ -871,229 +1064,377 @@ Load a batch import as a queued JSRPC job =cut sub process_batch_import { - my $job = shift; + my ($job, $param) = @_; - my $param = thaw(decode_base64(shift)); - my $format = $param->{'format'}; #well... this is all cch specific + if ( $param->{reload} ) { + process_batch_reload($job, $param); + } else { + # '_perform', yuck + _perform_batch_import($job, $param); + } - my $files = $param->{'uploaded_files'} - or die "No files provided."; +} - my (%files) = map { /^(\w+):([\.\w]+)$/ ? ($1,$2):() } split /,/, $files; +sub _perform_batch_import { + my ($job, $param) = @_; - if ($format eq 'cch' || $format eq 'cch-fixed') { + my $oldAutoCommit = $FS::UID::AutoCommit; + local $FS::UID::AutoCommit = 0; + my $dbh = dbh; + + my $format = $param->{'format'}; - my $oldAutoCommit = $FS::UID::AutoCommit; - local $FS::UID::AutoCommit = 0; - my $dbh = dbh; - my $error = ''; - my $have_location = 0; - - my @list = ( 'GEOCODE', 'geofile', \&FS::tax_rate_location::batch_import, - 'CODE', 'codefile', \&FS::tax_class::batch_import, - 'PLUS4', 'plus4file', \&FS::cust_tax_location::batch_import, - 'ZIP', 'zipfile', \&FS::cust_tax_location::batch_import, - 'TXMATRIX', 'txmatrix', \&FS::part_pkg_taxrate::batch_import, - 'DETAIL', 'detail', \&FS::tax_rate::batch_import, - ); - while( scalar(@list) ) { - my ($name, $file, $import_sub) = (shift @list, shift @list, shift @list); - unless ($files{$file}) { - next if $name eq 'PLUS4'; - $error = "No $name supplied"; - $error = "Neither PLUS4 nor ZIP supplied" - if ($name eq 'ZIP' && !$have_location); - next; - } - $have_location = 1 if $name eq 'PLUS4'; - my $fmt = $format. ( $name eq 'ZIP' ? '-zip' : '' ); - my $dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc; - my $filename = "$dir/". $files{$file}; - open my $fh, "< $filename" or $error ||= "Can't open $name file: $!"; + my $files = $param->{'uploaded_files'} + or die "No files provided."; - $error ||= &{$import_sub}({ 'filehandle' => $fh, 'format' => $fmt }, $job); - close $fh; - unlink $filename or warn "Can't delete $filename: $!"; - } - - if ($error) { - $dbh->rollback or die $dbh->errstr if $oldAutoCommit; - die $error; - }else{ - $dbh->commit or die $dbh->errstr if $oldAutoCommit; - } + my (%files) = map { /^(\w+):((taxdata\/\w+\.\w+\/)?[\.\w]+)$/ ? ($1,$2):() } + split /,/, $files; - }elsif ($format eq 'cch-update' || $format eq 'cch-fixed-update') { + my $dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc; + my $error = ''; + + if ( $format eq 'cch' || $format eq 'cch-fixed' + || $format eq 'cch-update' || $format eq 'cch-fixed-update' ) + { - my $oldAutoCommit = $FS::UID::AutoCommit; - local $FS::UID::AutoCommit = 0; - my $dbh = dbh; - my $error = ''; my @insert_list = (); my @delete_list = (); my @predelete_list = (); - - my @list = ( 'GEOCODE', 'geofile', \&FS::tax_rate_location::batch_import, - 'CODE', 'codefile', \&FS::tax_class::batch_import, - 'PLUS4', 'plus4file', \&FS::cust_tax_location::batch_import, - 'ZIP', 'zipfile', \&FS::cust_tax_location::batch_import, - 'TXMATRIX', 'txmatrix', \&FS::part_pkg_taxrate::batch_import, + my $insertname = ''; + my $deletename = ''; + + my @list = ( 'GEOCODE', \&FS::tax_rate_location::batch_import, + 'CODE', \&FS::tax_class::batch_import, + 'PLUS4', \&FS::cust_tax_location::batch_import, + 'ZIP', \&FS::cust_tax_location::batch_import, + 'TXMATRIX', \&FS::part_pkg_taxrate::batch_import, + 'DETAIL', \&FS::tax_rate::batch_import, ); - my $dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc; while( scalar(@list) ) { - my ($name, $file, $import_sub) = (shift @list, shift @list, shift @list); + my ( $name, $import_sub ) = splice( @list, 0, 2 ); + my $file = lc($name). 'file'; + unless ($files{$file}) { - my $vendor = $name eq 'ZIP' ? 'cch' : 'cch-zip'; - next # update expected only for previously installed location data - if ( ($name eq 'PLUS4' || $name eq 'ZIP') - && !scalar( qsearch( { table => 'cust_tax_location', - hashref => { data_vendor => $vendor }, - select => 'DISTINCT data_vendor', - } ) - ) - ); - - $error = "No $name supplied"; + #$error = "No $name supplied"; next; } + next if $name eq 'DETAIL' && $format =~ /update/; + my $filename = "$dir/". $files{$file}; - open my $fh, "< $filename" or $error ||= "Can't open $name file $filename: $!"; - unlink $filename or warn "Can't delete $filename: $!"; - - my $ifh = new File::Temp( TEMPLATE => "$name.insert.XXXXXXXX", - DIR => $dir, - UNLINK => 0, #meh - ) or die "can't open temp file: $!\n"; - - my $dfh = new File::Temp( TEMPLATE => "$name.delete.XXXXXXXX", - DIR => $dir, - UNLINK => 0, #meh - ) or die "can't open temp file: $!\n"; - - my $insert_pattern = ($format eq 'cch-update') ? qr/"I"\s*$/ : qr/I\s*$/; - my $delete_pattern = ($format eq 'cch-update') ? qr/"D"\s*$/ : qr/D\s*$/; - while(<$fh>) { - my $handle = ''; - $handle = $ifh if $_ =~ /$insert_pattern/; - $handle = $dfh if $_ =~ /$delete_pattern/; - unless ($handle) { - $error = "bad input line: $_" unless $handle; - last; + + if ( $format =~ /update/ ) { + + ( $error, $insertname, $deletename ) = + _perform_cch_insert_delete_split( $name, $filename, $dir, $format ) + unless $error; + last if $error; + + unlink $filename or warn "Can't delete $filename: $!" + unless $keep_cch_files; + push @insert_list, $name, $insertname, $import_sub, $format; + if ( $name eq 'GEOCODE' || $name eq 'CODE' ) { #handle this whole ordering issue better + unshift @predelete_list, $name, $deletename, $import_sub, $format; + } else { + unshift @delete_list, $name, $deletename, $import_sub, $format; } - print $handle $_; - } - close $fh; - close $ifh; - close $dfh; - push @insert_list, $name, $ifh->filename, $import_sub; - if ( $name eq 'GEOCODE' ) { #handle this whole ordering issue better - unshift @predelete_list, $name, $dfh->filename, $import_sub; } else { - unshift @delete_list, $name, $dfh->filename, $import_sub; + + push @insert_list, $name, $filename, $import_sub, $format; + } } - while( scalar(@predelete_list) ) { - my ($name, $file, $import_sub) = - (shift @predelete_list, shift @predelete_list, shift @predelete_list); + push @insert_list, + 'DETAIL', "$dir/".$files{detailfile}, \&FS::tax_rate::batch_import, $format + if $format =~ /update/; - my $fmt = $format. ( $name eq 'ZIP' ? '-zip' : '' ); - open my $fh, "< $file" or $error ||= "Can't open $name file $file: $!"; - $error ||= - &{$import_sub}({ 'filehandle' => $fh, 'format' => $fmt }, $job); - close $fh; - unlink $file or warn "Can't delete $file: $!"; + my %addl_param = (); + if ( $param->{'delete_only'} ) { + $addl_param{'delete_only'} = $param->{'delete_only'}; + @insert_list = () } - - while( scalar(@insert_list) ) { - my ($name, $file, $import_sub) = - (shift @insert_list, shift @insert_list, shift @insert_list); - my $fmt = $format. ( $name eq 'ZIP' ? '-zip' : '' ); - open my $fh, "< $file" or $error ||= "Can't open $name file $file: $!"; - $error ||= - &{$import_sub}({ 'filehandle' => $fh, 'format' => $fmt }, $job); - close $fh; + $error ||= _perform_cch_tax_import( $job, + [ @predelete_list ], + [ @insert_list ], + [ @delete_list ], + \%addl_param, + ); + + + @list = ( @predelete_list, @insert_list, @delete_list ); + while( !$keep_cch_files && scalar(@list) ) { + my ( undef, $file, undef, undef ) = splice( @list, 0, 4 ); unlink $file or warn "Can't delete $file: $!"; } - - $error ||= "No DETAIL supplied" - unless ($files{detail}); - open my $fh, "< $dir/". $files{detail} - or $error ||= "Can't open DETAIL file: $!"; - $error ||= - &FS::tax_rate::batch_import({ 'filehandle' => $fh, 'format' => $format }, - $job); - close $fh; - unlink "$dir/". $files{detail} or warn "Can't delete $files{detail}: $!" - if $files{detail}; - while( scalar(@delete_list) ) { - my ($name, $file, $import_sub) = - (shift @delete_list, shift @delete_list, shift @delete_list); + } elsif ( $format =~ /^billsoft-(\w+)$/ ) { + my $mode = $1; + my $file = $dir.'/'.$files{'file'}; + open my $fh, "< $file" or $error ||= "Can't open file $file: $!"; + my @param = ( + { + filehandle => $fh, + format => 'billsoft', + }, $job); + if ( $mode eq 'pcode' ) { + $error ||= FS::cust_tax_location::batch_import(@param); + seek $fh, 0, 0; + $error ||= FS::tax_rate_location::batch_import(@param); + } elsif ( $mode eq 'taxclass' ) { + $error ||= FS::tax_class::batch_import(@param); + } elsif ( $mode eq 'taxproduct' ) { + $error ||= FS::part_pkg_taxproduct::batch_import(@param); + } else { + die "unknown import mode 'billsoft-$mode'\n"; + } - my $fmt = $format. ( $name eq 'ZIP' ? '-zip' : '' ); + } else { + die "Unknown format: $format"; + } + + if ($error) { + $dbh->rollback or die $dbh->errstr if $oldAutoCommit; + die $error; + } else { + $dbh->commit or die $dbh->errstr if $oldAutoCommit; + } + +} + +# +# +# EVERYTHING THAT FOLLOWS IS CCH-SPECIFIC. +# +# + +sub _perform_cch_tax_import { + my ( $job, $predelete_list, $insert_list, $delete_list, $addl_param ) = @_; + $addl_param ||= {}; + + my $error = ''; + foreach my $list ($predelete_list, $insert_list, $delete_list) { + while( scalar(@$list) ) { + my ( $name, $file, $method, $format ) = splice( @$list, 0, 4 ); + my $fmt = "$format-update"; + $fmt = $format. ( lc($name) eq 'zip' ? '-zip' : '' ); open my $fh, "< $file" or $error ||= "Can't open $name file $file: $!"; - $error ||= - &{$import_sub}({ 'filehandle' => $fh, 'format' => $fmt }, $job); + my $param = { 'filehandle' => $fh, + 'format' => $fmt, + %$addl_param, + }; + $error ||= &{$method}($param, $job); close $fh; - unlink $file or warn "Can't delete $file: $!"; - } - - if ($error) { - $dbh->rollback or die $dbh->errstr if $oldAutoCommit; - die $error; - }else{ - $dbh->commit or die $dbh->errstr if $oldAutoCommit; } + } - }else{ - die "Unknown format: $format"; + return $error; +} + +sub _perform_cch_insert_delete_split { + my ($name, $filename, $dir, $format) = @_; + + my $error = ''; + + open my $fh, "< $filename" + or $error ||= "Can't open $name file $filename: $!"; + + my $ifh = new File::Temp( TEMPLATE => "$name.insert.XXXXXXXX", + DIR => $dir, + UNLINK => 0, #meh + ) or die "can't open temp file: $!\n"; + my $insertname = $ifh->filename; + + my $dfh = new File::Temp( TEMPLATE => "$name.delete.XXXXXXXX", + DIR => $dir, + UNLINK => 0, #meh + ) or die "can't open temp file: $!\n"; + my $deletename = $dfh->filename; + + my $insert_pattern = ($format eq 'cch-update') ? qr/"I"\s*$/ : qr/I\s*$/; + my $delete_pattern = ($format eq 'cch-update') ? qr/"D"\s*$/ : qr/D\s*$/; + while(<$fh>) { + my $handle = ''; + $handle = $ifh if $_ =~ /$insert_pattern/; + $handle = $dfh if $_ =~ /$delete_pattern/; + unless ($handle) { + $error = "bad input line: $_" unless $handle; + last; + } + print $handle $_; } + close $fh; + close $ifh; + close $dfh; + return ($error, $insertname, $deletename); } -=item process_download_and_reload +sub _perform_cch_diff { + my ($name, $newdir, $olddir) = @_; -Download and process a tax update as a queued JSRPC job after wiping the -existing wipable tax data. + my %oldlines = (); -=cut + if ($olddir) { + open my $oldcsvfh, "$olddir/$name.txt" + or die "failed to open $olddir/$name.txt: $!\n"; -sub process_download_and_reload { - my $job = shift; + while(<$oldcsvfh>) { + chomp; + $oldlines{$_} = 1; + } + close $oldcsvfh; + } - my $param = thaw(decode_base64($_[0])); - my $format = $param->{'format'}; #well... this is all cch specific + open my $newcsvfh, "$newdir/$name.txt" + or die "failed to open $newdir/$name.txt: $!\n"; + + my $dfh = new File::Temp( TEMPLATE => "$name.diff.XXXXXXXX", + DIR => "$newdir", + UNLINK => 0, #meh + ) or die "can't open temp file: $!\n"; + my $diffname = $dfh->filename; + + while(<$newcsvfh>) { + chomp; + if (exists($oldlines{$_})) { + $oldlines{$_} = 0; + } else { + print $dfh $_, ',"I"', "\n"; + } + } + close $newcsvfh; - my ( $count, $last, $min_sec, $imported ) = (0, time, 5, 0); #progressbar - $count = 100; + #false laziness w/above (sub batch_import) + my @fields = qw( geocode inoutcity inoutlocal tax location taxbase taxmax + excessrate effective_date taxauth taxtype taxcat taxname + usetax useexcessrate fee unittype feemax maxtype passflag + passtype basetype ); + my $numfields = scalar(@fields); - if ( $job ) { # progress bar - my $error = $job->update_statustext( int( 100 * $imported / $count ) ); + my $csv = new Text::CSV_XS { 'always_quote' => 1 }; + + for my $line (grep $oldlines{$_}, keys %oldlines) { + + $csv->parse($line) or do { + #$dbh->rollback if $oldAutoCommit; + die "can't parse: ". $csv->error_input(); + }; + my @columns = $csv->fields(); + + $csv->combine( splice(@columns, 0, $numfields) ); + + print $dfh $csv->string, ',"D"', "\n"; + } + + close $dfh; + + return $diffname; +} + +sub _cch_fetch_and_unzip { + my ( $job, $urls, $secret, $dir ) = @_; + + my $ua = new LWP::UserAgent; + foreach my $url (split ',', $urls) { + my @name = split '/', $url; #somewhat restrictive + my $name = pop @name; + $name =~ /([\w.]+)/; # untaint that which we don't trust so much any more + $name = $1; + + open my $taxfh, ">$dir/$name" or die "Can't open $dir/$name: $!\n"; + + my ( $imported, $last, $min_sec ) = _progressbar_foo(); + my $res = $ua->request( + new HTTP::Request( GET => $url ), + sub { + print $taxfh $_[0] or die "Can't write to $dir/$name: $!\n"; + my $content_length = $_[1]->content_length; + $imported += length($_[0]); + if ( time - $min_sec > $last ) { + my $error = $job->update_statustext( + ($content_length ? int(100 * $imported/$content_length) : 0 ). + ",Downloading data from CCH" + ); + die $error if $error; + $last = time; + } + }, + ); + die "download of $url failed: ". $res->status_line + unless $res->is_success; + + close $taxfh; + my $error = $job->update_statustext( "0,Unpacking data" ); die $error if $error; + $secret =~ /([\w.]+)/; # untaint that which we don't trust so much any more + $secret = $1; + system('unzip', "-P", $secret, "-d", "$dir", "$dir/$name") == 0 + or die "unzip -P $secret -d $dir $dir/$name failed"; + #unlink "$dir/$name"; } +} + +sub _cch_extract_csv_from_dbf { + my ( $job, $dir, $name ) = @_; - my $oldAutoCommit = $FS::UID::AutoCommit; - local $FS::UID::AutoCommit = 0; - my $dbh = dbh; - my $error = ''; + eval "use XBase;"; + die $@ if $@; - my $sql = - "SELECT count(*) FROM part_pkg_taxoverride JOIN tax_class ". - "USING (taxclassnum) WHERE data_vendor = '$format'"; - my $sth = $dbh->prepare($sql) or die $dbh->errstr; - $sth->execute - or die "Unexpected error executing statement $sql: ". $sth->errstr; - die "Don't (yet) know how to handle part_pkg_taxoverride records." - if $sth->fetchrow_arrayref->[0]; + my ( $imported, $last, $min_sec ) = _progressbar_foo(); + my $error = $job->update_statustext( "0,Unpacking $name" ); + die $error if $error; + warn "opening $dir.new/$name.dbf\n" if $DEBUG; + my $table = new XBase 'name' => "$dir.new/$name.dbf"; + die "failed to access $dir.new/$name.dbf: ". XBase->errstr + unless defined($table); + my $count = $table->last_record; # approximately; + open my $csvfh, ">$dir.new/$name.txt" + or die "failed to open $dir.new/$name.txt: $!\n"; + + my $csv = new Text::CSV_XS { 'always_quote' => 1 }; + my @fields = $table->field_names; + my $cursor = $table->prepare_select; + my $format_date = + sub { my $date = shift; + $date =~ /^(\d{4})(\d{2})(\d{2})$/ && ($date = "$2/$3/$1"); + $date; + }; + while (my $row = $cursor->fetch_hashref) { + $csv->combine( map { my $type = $table->field_type($_); + if ($type eq 'D') { + &{$format_date}($row->{$_}) ; + } elsif ($type eq 'N' && $row->{$_} =~ /e-/i ) { + sprintf('%.8f', $row->{$_}); #db row is numeric(14,8) + } else { + $row->{$_}; + } + } + @fields + ); + print $csvfh $csv->string, "\n"; + $imported++; + if ( time - $min_sec > $last ) { + my $error = $job->update_statustext( + int(100 * $imported/$count). ",Unpacking $name" + ); + die $error if $error; + $last = time; + } + } + $table->close; + close $csvfh; +} - # really should get a table EXCLUSIVE lock here +sub _remember_disabled_taxes { + my ( $job, $format, $disabled_tax_rate ) = @_; + + # cch specific hash + + my ( $imported, $last, $min_sec ) = _progressbar_foo(); - #remember disabled taxes - my %disabled_tax_rate = (); my @items = qsearch( { table => 'tax_rate', hashref => { disabled => 'Y', data_vendor => $format, @@ -1101,16 +1442,12 @@ sub process_download_and_reload { select => 'geocode, taxclassnum', } ); - $count = scalar(@items); + my $count = scalar(@items); foreach my $tax_rate ( @items ) { if ( time - $min_sec > $last ) { - my $error = $job->update_statustext( + $job->update_statustext( int( 100 * $imported / $count ). ",Remembering disabled taxes" ); - if ($error) { - $dbh->rollback or die $dbh->errstr if $oldAutoCommit; - die $error; - } $last = time; } $imported++; @@ -1120,148 +1457,78 @@ sub process_download_and_reload { warn "failed to find tax_class ". $tax_rate->taxclassnum; next; } - $disabled_tax_rate{$tax_rate->geocode. ':'. $tax_class->taxclass} = 1; + $disabled_tax_rate->{$tax_rate->geocode. ':'. $tax_class->taxclass} = 1; } +} + +sub _remember_tax_products { + my ( $job, $format, $taxproduct ) = @_; - #remember tax products # XXX FIXME this loop only works when cch is the only data provider - my %taxproduct = (); - my $extra_sql = "WHERE taxproductnum IS NOT NULL OR ". - "0 < ( SELECT count(*) from part_pkg_option WHERE ". - " part_pkg_option.pkgpart = part_pkg.pkgpart AND ". - " optionname LIKE 'usage_taxproductnum_%' AND ". - " optionvalue != '' )"; - @items = qsearch( { table => 'part_pkg', - select => 'DISTINCT pkgpart,taxproductnum', - hashref => {}, - extra_sql => $extra_sql, - } - ); - $count = scalar(@items); - $imported = 0; + + my ( $imported, $last, $min_sec ) = _progressbar_foo(); + + my $extra_sql = " + WHERE taxproductnum IS NOT NULL + OR EXISTS ( SELECT 1 from part_pkg_option + WHERE part_pkg_option.pkgpart = part_pkg.pkgpart + AND optionname LIKE 'usage_taxproductnum_%' + AND optionvalue != '' + ) + "; + my @items = qsearch( { table => 'part_pkg', + select => 'DISTINCT pkgpart,taxproductnum', + hashref => {}, + extra_sql => $extra_sql, + } + ); + my $count = scalar(@items); foreach my $part_pkg ( @items ) { if ( time - $min_sec > $last ) { - my $error = $job->update_statustext( + $job->update_statustext( int( 100 * $imported / $count ). ",Remembering tax products" ); - if ($error) { - $dbh->rollback or die $dbh->errstr if $oldAutoCommit; - die $error; - } $last = time; } $imported++; warn "working with package part ". $part_pkg->pkgpart. "which has a taxproductnum of ". $part_pkg->taxproductnum. "\n" if $DEBUG; my $part_pkg_taxproduct = $part_pkg->taxproduct(''); - $taxproduct{$part_pkg->pkgpart}{''} = $part_pkg_taxproduct->taxproduct - if $part_pkg_taxproduct; + $taxproduct->{$part_pkg->pkgpart}->{''} = $part_pkg_taxproduct->taxproduct + if $part_pkg_taxproduct && $part_pkg_taxproduct->data_vendor eq $format; foreach my $option ( $part_pkg->part_pkg_option ) { - next unless $option->optionname =~ /^usage_taxproductnum_(\w)$/; + next unless $option->optionname =~ /^usage_taxproductnum_(\w+)$/; my $class = $1; $part_pkg_taxproduct = $part_pkg->taxproduct($class); - $taxproduct{$part_pkg->pkgpart}{$class} = $part_pkg_taxproduct->taxproduct - if $part_pkg_taxproduct; - } - } - - #wipe out the old data - $error = $job->update_statustext( "0,Removing old tax data" ); - if ($error) { - $dbh->rollback or die $dbh->errstr if $oldAutoCommit; - die $error; - } - foreach my $tax_rate_location ( qsearch( 'tax_rate_location', - { data_vendor => $format, - disabled => '', - } - ) - ) - { - $tax_rate_location->disabled('Y'); - my $error = $tax_rate_location->replace; - if ( $error ) { - $dbh->rollback or die $dbh->errstr if $oldAutoCommit; - die $error; - } - } - - local $FS::part_pkg_taxproduct::delete_kludge = 1; - my @table = qw( - tax_rate part_pkg_taxrate part_pkg_taxproduct tax_class cust_tax_location - ); - foreach my $table ( @table ) { - my $dbh = dbh; -# my $primary_key = dbdef->table($table)->primary_key; -# my $sql = "SELECT $primary_key FROM $table WHERE data_vendor = ". - my $sql = "DELETE FROM $table WHERE data_vendor = ". - $dbh->quote($format); - my $sth = $dbh->prepare($sql); - unless ($sth) { - $error = $dbh->errstr; - $dbh->rollback or die $dbh->errstr if $oldAutoCommit; - die $error; + $taxproduct->{$part_pkg->pkgpart}->{$class} = + $part_pkg_taxproduct->taxproduct + if $part_pkg_taxproduct && $part_pkg_taxproduct->data_vendor eq $format; } - unless ($sth->execute) { - $dbh->rollback or die $dbh->errstr if $oldAutoCommit; - die "Failed to execute $sql: ". $sth->errstr; - } -# foreach my $row ( @{ $sth->fetchall_arrayref } ) { -# my $record = qsearchs( $table, { $primary_key => $row->[0] } ) -# or die "Failed to find $table with $primary_key ". $row->[0]; -# my $error = $record->delete; -# if ( $error ) { -# $dbh->rollback or die $dbh->errstr if $oldAutoCommit; -# die $error; -# } -# } } +} - if ( $format eq 'cch' ) { - foreach my $cust_tax_location ( qsearch( 'cust_tax_location', - { data_vendor => "$format-zip" } - ) - ) - { - my $error = $cust_tax_location->delete; - if ( $error ) { - $dbh->rollback or die $dbh->errstr if $oldAutoCommit; - die $error; - } - } - } +sub _restore_remembered_tax_products { + my ( $job, $format, $taxproduct ) = @_; - #import new data - my $statement = ' &process_download_and_update($job, @_); '; - eval $statement; - if ($@) { - $dbh->rollback or die $dbh->errstr if $oldAutoCommit; - die $@; - } + # cch specific - #restore taxproducts - $count = scalar(keys %taxproduct); - $imported = 0; - foreach my $pkgpart ( keys %taxproduct ) { + my ( $imported, $last, $min_sec ) = _progressbar_foo(); + my $count = scalar(keys %$taxproduct); + foreach my $pkgpart ( keys %$taxproduct ) { warn "restoring taxproductnums on pkgpart $pkgpart\n" if $DEBUG; if ( time - $min_sec > $last ) { - my $error = $job->update_statustext( + $job->update_statustext( int( 100 * $imported / $count ). ",Restoring tax products" ); - if ( $error ) { - $dbh->rollback or die $dbh->errstr if $oldAutoCommit; - die $error; - } $last = time; } $imported++; my $part_pkg = qsearchs('part_pkg', { pkgpart => $pkgpart } ); unless ( $part_pkg ) { - $dbh->rollback or die $dbh->errstr if $oldAutoCommit; - die "somehow failed to find part_pkg with pkgpart $pkgpart!\n"; + return "somehow failed to find part_pkg with pkgpart $pkgpart!\n"; } my %options = $part_pkg->options; @@ -1269,19 +1536,18 @@ sub process_download_and_reload { my $primary_svc = $part_pkg->svcpart; my $new = new FS::part_pkg { $part_pkg->hash }; - foreach my $class ( keys %{ $taxproduct{$pkgpart} } ) { + foreach my $class ( keys %{ $taxproduct->{$pkgpart} } ) { warn "working with class '$class'\n" if $DEBUG; my $part_pkg_taxproduct = qsearchs( 'part_pkg_taxproduct', - { taxproduct => $taxproduct{$pkgpart}{$class}, + { taxproduct => $taxproduct->{$pkgpart}->{$class}, data_vendor => $format, } ); unless ( $part_pkg_taxproduct ) { - $dbh->rollback or die $dbh->errstr if $oldAutoCommit; - die "failed to find part_pkg_taxproduct ($taxproduct{pkgpart}{$class})". - " for pkgpart $pkgpart\n"; + return "failed to find part_pkg_taxproduct (". + $taxproduct->{$pkgpart}->{$class}. ") for pkgpart $pkgpart\n"; } if ( $class eq '' ) { @@ -1300,24 +1566,23 @@ sub process_download_and_reload { 'options' => \%options, ); - if ( $error ) { - $dbh->rollback or die $dbh->errstr if $oldAutoCommit; - die $error; - } + return $error if $error; + } - #disable tax_rates - $count = scalar(keys %disabled_tax_rate); - $imported = 0; - foreach my $key (keys %disabled_tax_rate) { + ''; +} + +sub _restore_remembered_disabled_taxes { + my ( $job, $format, $disabled_tax_rate ) = @_; + + my ( $imported, $last, $min_sec ) = _progressbar_foo(); + my $count = scalar(keys %$disabled_tax_rate); + foreach my $key (keys %$disabled_tax_rate) { if ( time - $min_sec > $last ) { - my $error = $job->update_statustext( + $job->update_statustext( int( 100 * $imported / $count ). ",Disabling tax rates" ); - if ( $error ) { - $dbh->rollback or die $dbh->errstr if $oldAutoCommit; - die $error; - } $last = time; } $imported++; @@ -1325,10 +1590,8 @@ sub process_download_and_reload { my @tax_class = qsearch( 'tax_class', { data_vendor => $format, taxclass => $taxclass, } ); - if (scalar(@tax_class) > 1) { - $dbh->rollback or die $dbh->errstr if $oldAutoCommit; - die "found multiple tax_class records for format $format class $taxclass"; - } + return "found multiple tax_class records for format $format class $taxclass" + if scalar(@tax_class) > 1; unless (scalar(@tax_class)) { warn "no tax_class for format $format class $taxclass\n"; @@ -1343,28 +1606,222 @@ sub process_download_and_reload { ); if (scalar(@tax_rate) > 1) { - $dbh->rollback or die $dbh->errstr if $oldAutoCommit; - die "found multiple tax_rate records for format $format geocode $geocode". - " and taxclass $taxclass ( taxclassnum ". $tax_class[0]->taxclassnum. - " )"; + return "found multiple tax_rate records for format $format geocode ". + "$geocode and taxclass $taxclass ( taxclassnum ". + $tax_class[0]->taxclassnum. " )"; } if (scalar(@tax_rate)) { $tax_rate[0]->disabled('Y'); my $error = $tax_rate[0]->replace; - if ( $error ) { - $dbh->rollback or die $dbh->errstr if $oldAutoCommit; - die $error; + return $error if $error; + } + } +} + +sub _remove_old_tax_data { + my ( $job, $format ) = @_; + + my $dbh = dbh; + my $error = $job->update_statustext( "0,Removing old tax data" ); + die $error if $error; + + my $sql = "UPDATE public.tax_rate_location SET disabled='Y' ". + "WHERE data_vendor = ". $dbh->quote($format); + $dbh->do($sql) or return "Failed to execute $sql: ". $dbh->errstr; + + my @table = qw( + tax_rate part_pkg_taxrate part_pkg_taxproduct tax_class cust_tax_location + ); + foreach my $table ( @table ) { + $sql = "DELETE FROM public.$table WHERE data_vendor = ". + $dbh->quote($format); + $dbh->do($sql) or return "Failed to execute $sql: ". $dbh->errstr; + } + + if ( $format eq 'cch' ) { + $sql = "DELETE FROM public.cust_tax_location WHERE data_vendor = ". + $dbh->quote("$format-zip"); + $dbh->do($sql) or return "Failed to execute $sql: ". $dbh->errstr; + } + + ''; +} + +sub _create_temporary_tables { + my ( $job, $format ) = @_; + + my $dbh = dbh; + my $error = $job->update_statustext( "0,Creating temporary tables" ); + die $error if $error; + + my @table = qw( tax_rate + tax_rate_location + part_pkg_taxrate + part_pkg_taxproduct + tax_class + cust_tax_location + ); + foreach my $table ( @table ) { + my $sql = + "CREATE TEMPORARY TABLE $table ( LIKE $table INCLUDING DEFAULTS )"; + $dbh->do($sql) or return "Failed to execute $sql: ". $dbh->errstr; + } + + ''; +} + +sub _copy_from_temp { + my ( $job, $format ) = @_; + + my $dbh = dbh; + my $error = $job->update_statustext( "0,Making permanent" ); + die $error if $error; + + my @table = qw( tax_rate + tax_rate_location + part_pkg_taxrate + part_pkg_taxproduct + tax_class + cust_tax_location + ); + foreach my $table ( @table ) { + my $sql = + "INSERT INTO public.$table SELECT * from $table"; + $dbh->do($sql) or return "Failed to execute $sql: ". $dbh->errstr; + } + + ''; +} + +=item process_download_and_reload + +Download and process a tax update as a queued JSRPC job after wiping the +existing wipeable tax data. + +=cut + +sub process_download_and_reload { + _process_reload(\&process_download_and_update, @_); +} + +# +# +# END OF CCH STUFF +# +# + +=item process_batch_reload + +Load and process a tax update from the provided files as a queued JSRPC job +after wiping the existing wipable tax data. + +=cut + +sub process_batch_reload { + _process_reload(\&_perform_batch_import, @_); +} + +sub _process_reload { + my ( $continuation, $job, $param ) = @_; + my $format = $param->{'format'}; + + my ( $imported, $last, $min_sec ) = _progressbar_foo(); + + if ( $job ) { # progress bar + my $error = $job->update_statustext( 0 ); + die $error if $error; + } + + my $oldAutoCommit = $FS::UID::AutoCommit; + local $FS::UID::AutoCommit = 0; + my $dbh = dbh; + my $error = ''; + + if ( $format =~ /^cch/ ) { + # no, THIS part is CCH specific + + my $sql = + "SELECT count(*) FROM part_pkg_taxoverride JOIN tax_class ". + "USING (taxclassnum) WHERE data_vendor = '$format'"; + my $sth = $dbh->prepare($sql) or die $dbh->errstr; + $sth->execute + or die "Unexpected error executing statement $sql: ". $sth->errstr; + die "Don't (yet) know how to handle part_pkg_taxoverride records." + if $sth->fetchrow_arrayref->[0]; + + # really should get a table EXCLUSIVE lock here + + #remember disabled taxes + my %disabled_tax_rate = (); + $error ||= _remember_disabled_taxes( $job, $format, \%disabled_tax_rate ); + + #remember tax products + my %taxproduct = (); + $error ||= _remember_tax_products( $job, $format, \%taxproduct ); + + #create temp tables + $error ||= _create_temporary_tables( $job, $format ); + + #import new data + unless ($error) { + eval { &{$continuation}( $job, $param ) }; + $error = $@ if $@; + } + + #restore taxproducts + $error ||= _restore_remembered_tax_products( $job, $format, \%taxproduct ); + + #disable tax_rates + $error ||= + _restore_remembered_disabled_taxes( $job, $format, \%disabled_tax_rate ); + + #wipe out the old data + $error ||= _remove_old_tax_data( $job, $format ); + + #untemporize + $error ||= _copy_from_temp( $job, $format ); + + } elsif ( $format =~ /^billsoft-(\w+)/ ) { + + my $mode = $1; + my @sql; + if ( $mode eq 'pcode' ) { + push @sql, + "DELETE FROM cust_tax_location WHERE data_vendor = 'billsoft'", + "UPDATE tax_rate_location SET disabled = 'Y' WHERE data_vendor = 'billsoft'"; + } elsif ( $mode eq 'taxclass' ) { + push @sql, + "DELETE FROM tax_class WHERE data_vendor = 'billsoft'"; + } elsif ( $mode eq 'taxproduct' ) { + push @sql, + "DELETE FROM part_pkg_taxproduct WHERE data_vendor = 'billsoft'"; + } + + foreach (@sql) { + if (!$dbh->do($_)) { + $error = $dbh->errstr; + last; } } + + unless ( $error ) { + local $@; + eval { &{ $continuation }($job, $param) }; + $error = $@; + } + } # if ($format ...) + + if ($error) { + $dbh->rollback or die $dbh->errstr if $oldAutoCommit; + die $error; } #success! - $dbh->commit or die $dbh->errstr if $oldAutoCommit; - } + =item process_download_and_update Download and process a tax update as a queued JSRPC job @@ -1374,29 +1831,25 @@ Download and process a tax update as a queued JSRPC job sub process_download_and_update { my $job = shift; - my $param = thaw(decode_base64(shift)); + my $param = shift; my $format = $param->{'format'}; #well... this is all cch specific - my ( $count, $last, $min_sec, $imported ) = (0, time, 5, 0); #progressbar - $count = 100; + my ( $imported, $last, $min_sec ) = _progressbar_foo(); if ( $job ) { # progress bar - my $error = $job->update_statustext( int( 100 * $imported / $count ) ); + my $error = $job->update_statustext( 0); die $error if $error; } - my $dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc. '/taxdata'; + my $cache_dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc. '/'; + my $dir = $cache_dir. 'taxdata'; unless (-d $dir) { mkdir $dir or die "can't create $dir: $!\n"; } if ($format eq 'cch') { - eval "use Text::CSV_XS;"; - die $@ if $@; - - eval "use XBase;"; - die $@ if $@; + my @namelist = qw( code detail geocode plus4 txmatrix zip ); my $conf = new FS::Conf; die "direct download of tax data not enabled\n" @@ -1409,33 +1862,33 @@ sub process_download_and_update { $dir .= '/cch'; - my $oldAutoCommit = $FS::UID::AutoCommit; - local $FS::UID::AutoCommit = 0; my $dbh = dbh; my $error = ''; # really should get a table EXCLUSIVE lock here # check if initial import or update + # + # relying on mkdir "$dir.new" as a mutex my $sql = "SELECT count(*) from tax_rate WHERE data_vendor='$format'"; my $sth = $dbh->prepare($sql) or die $dbh->errstr; $sth->execute() or die $sth->errstr; - my $upgrade = $sth->fetchrow_arrayref->[0]; + my $update = $sth->fetchrow_arrayref->[0]; # create cache and/or rotate old tax data if (-d $dir) { - if (-d "$dir.4") { - opendir(my $dirh, "$dir.4") or die "failed to open $dir.4: $!\n"; + if (-d "$dir.9") { + opendir(my $dirh, "$dir.9") or die "failed to open $dir.9: $!\n"; foreach my $file (readdir($dirh)) { - unlink "$dir.4/$file" if (-f "$dir.4/$file"); + unlink "$dir.9/$file" if (-f "$dir.9/$file"); } closedir($dirh); - rmdir "$dir.4"; + rmdir "$dir.9"; } - for (3, 2, 1) { + for (8, 7, 6, 5, 4, 3, 2, 1) { if ( -e "$dir.$_" ) { rename "$dir.$_", "$dir.". ($_+1) or die "can't rename $dir.$_: $!\n"; } @@ -1444,7 +1897,7 @@ sub process_download_and_update { } else { - die "can't find previous tax data\n" if $upgrade; + die "can't find previous tax data\n" if $update; } @@ -1452,215 +1905,37 @@ sub process_download_and_update { # fetch and unpack the zip files - my $ua = new LWP::UserAgent; - foreach my $url (split ',', $urls) { - my @name = split '/', $url; #somewhat restrictive - my $name = pop @name; - $name =~ /(.*)/; # untaint that which we trust; - $name = $1; - - open my $taxfh, ">$dir.new/$name" or die "Can't open $dir.new/$name: $!\n"; - - my $res = $ua->request( - new HTTP::Request( GET => $url), - sub { #my ($data, $response_object) = @_; - print $taxfh $_[0] or die "Can't write to $dir.new/$name: $!\n"; - my $content_length = $_[1]->content_length; - $imported += length($_[0]); - if ( time - $min_sec > $last ) { - my $error = $job->update_statustext( - ($content_length ? int(100 * $imported/$content_length) : 0 ). - ",Downloading data from CCH" - ); - die $error if $error; - $last = time; - } - }, - ); - die "download of $url failed: ". $res->status_line - unless $res->is_success; - - close $taxfh; - my $error = $job->update_statustext( "0,Unpacking data" ); - die $error if $error; - $secret =~ /(.*)/; # untaint that which we trust; - $secret = $1; - system('unzip', "-P", $secret, "-d", "$dir.new", "$dir.new/$name") == 0 - or die "unzip -P $secret -d $dir.new $dir.new/$name failed"; - #unlink "$dir.new/$name"; - } + _cch_fetch_and_unzip( $job, $urls, $secret, "$dir.new" ); # extract csv files from the dbf files - foreach my $name ( qw( code detail geocode plus4 txmatrix zip ) ) { - my $error = $job->update_statustext( "0,Unpacking $name" ); - die $error if $error; - warn "opening $dir.new/$name.dbf\n" if $DEBUG; - my $table = new XBase 'name' => "$dir.new/$name.dbf"; - die "failed to access $dir.new/$name.dbf: ". XBase->errstr - unless defined($table); - $count = $table->last_record; # approximately; - $imported = 0; - open my $csvfh, ">$dir.new/$name.txt" - or die "failed to open $dir.new/$name.txt: $!\n"; - - my $csv = new Text::CSV_XS { 'always_quote' => 1 }; - my @fields = $table->field_names; - my $cursor = $table->prepare_select; - my $format_date = - sub { my $date = shift; - $date =~ /^(\d{4})(\d{2})(\d{2})$/ && ($date = "$2/$3/$1"); - $date; - }; - while (my $row = $cursor->fetch_hashref) { - $csv->combine( map { ($table->field_type($_) eq 'D') - ? &{$format_date}($row->{$_}) - : $row->{$_} - } - @fields - ); - print $csvfh $csv->string, "\n"; - $imported++; - if ( time - $min_sec > $last ) { - my $error = $job->update_statustext( - int(100 * $imported/$count). ",Unpacking $name" - ); - die $error if $error; - $last = time; - } - } - $table->close; - close $csvfh; + foreach my $name ( @namelist ) { + _cch_extract_csv_from_dbf( $job, $dir, $name ); } # generate the diff files - my @insert_list = (); - my @delete_list = (); - my @predelete_list = (); - - my @list = ( - 'geocode', \&FS::tax_rate_location::batch_import, - 'code', \&FS::tax_class::batch_import, - 'plus4', \&FS::cust_tax_location::batch_import, - 'zip', \&FS::cust_tax_location::batch_import, - 'txmatrix', \&FS::part_pkg_taxrate::batch_import, - 'detail', \&FS::tax_rate::batch_import, - ); - - while( scalar(@list) ) { - my ( $name, $method ) = ( shift @list, shift @list ); - my %oldlines = (); - - my $error = $job->update_statustext( "0,Comparing to previous $name" ); - die $error if $error; - - warn "processing $dir.new/$name.txt\n" if $DEBUG; - - if ($upgrade) { - open my $oldcsvfh, "$dir.1/$name.txt" - or die "failed to open $dir.1/$name.txt: $!\n"; - - while(<$oldcsvfh>) { - chomp; - $oldlines{$_} = 1; - } - close $oldcsvfh; + my @list = (); + foreach my $name ( @namelist ) { + my $difffile = "$dir.new/$name.txt"; + if ($update) { + my $error = $job->update_statustext( "0,Comparing to previous $name" ); + die $error if $error; + warn "processing $dir.new/$name.txt\n" if $DEBUG; + my $olddir = $update ? "$dir.1" : ""; + $difffile = _perform_cch_diff( $name, "$dir.new", $olddir ); } - - open my $newcsvfh, "$dir.new/$name.txt" - or die "failed to open $dir.new/$name.txt: $!\n"; - - my $ifh = new File::Temp( TEMPLATE => "$name.insert.XXXXXXXX", - DIR => "$dir.new", - UNLINK => 0, #meh - ) or die "can't open temp file: $!\n"; - - my $dfh = new File::Temp( TEMPLATE => "$name.delete.XXXXXXXX", - DIR => "$dir.new", - UNLINK => 0, #meh - ) or die "can't open temp file: $!\n"; - - while(<$newcsvfh>) { - chomp; - if (exists($oldlines{$_})) { - $oldlines{$_} = 0; - } else { - print $ifh $_, ',"I"', "\n"; - } - } - close $newcsvfh; - - if ($name eq 'detail') { - for (keys %oldlines) { # one file for rate details - print $ifh $_, ',"D"', "\n" if $oldlines{$_}; - } - } else { - for (keys %oldlines) { - print $dfh $_, ',"D"', "\n" if $oldlines{$_}; - } - } - %oldlines = (); - - push @insert_list, $name, $ifh->filename, $method; - if ( $name eq 'geocode' ) { - unshift @predelete_list, $name, $dfh->filename, $method - unless $name eq 'detail'; - } else { - unshift @delete_list, $name, $dfh->filename, $method - unless $name eq 'detail'; - } - - close $dfh; - close $ifh; + $difffile =~ s/^$cache_dir//; + push @list, "${name}file:$difffile"; } - while( scalar(@predelete_list) ) { - my ($name, $file, $method) = - (shift @predelete_list, shift @predelete_list, shift @predelete_list); - - my $fmt = "$format-update"; - $fmt = $fmt. ( $name eq 'zip' ? '-zip' : '' ); - open my $fh, "< $file" or $error ||= "Can't open $name file $file: $!"; - $error ||= - &{$method}({ 'filehandle' => $fh, 'format' => $fmt }, $job); - close $fh; - #unlink $file or warn "Can't delete $file: $!"; - } - - while( scalar(@insert_list) ) { - my ($name, $file, $method) = - (shift @insert_list, shift @insert_list, shift @insert_list); - - my $fmt = "$format-update"; - $fmt = $fmt. ( $name eq 'zip' ? '-zip' : '' ); - open my $fh, "< $file" or $error ||= "Can't open $name file $file: $!"; - $error ||= - &{$method}({ 'filehandle' => $fh, 'format' => $fmt }, $job); - close $fh; - #unlink $file or warn "Can't delete $file: $!"; - } - - while( scalar(@delete_list) ) { - my ($name, $file, $method) = - (shift @delete_list, shift @delete_list, shift @delete_list); - - my $fmt = "$format-update"; - $fmt = $fmt. ( $name eq 'zip' ? '-zip' : '' ); - open my $fh, "< $file" or $error ||= "Can't open $name file $file: $!"; - $error ||= - &{$method}({ 'filehandle' => $fh, 'format' => $fmt }, $job); - close $fh; - #unlink $file or warn "Can't delete $file: $!"; - } + # perform the import + local $keep_cch_files = 1; + $param->{uploaded_files} = join( ',', @list ); + $param->{format} .= '-update' if $update; + $error ||= + _perform_batch_import( $job, $param ); - if ($error) { - $dbh->rollback or die $dbh->errstr if $oldAutoCommit; - die $error; - }else{ - $dbh->commit or die $dbh->errstr if $oldAutoCommit; - } - rename "$dir.new", "$dir" or die "cch tax update processed, but can't rename $dir.new: $!\n"; @@ -1749,121 +2024,335 @@ sub browse_queries { return ($query, "SELECT COUNT(*) FROM tax_rate $extra_sql"); } -# _upgrade_data -# -# Used by FS::Upgrade to migrate to a new database. -# -# +=item queue_liability_report PARAMS -sub _upgrade_data { # class method - my ($self, %opts) = @_; - my $dbh = dbh; +Launches a tax liability report. + +PARAMS needs to be a base64-encoded Storable hash containing: +- beginning: the start date, as a I (not a timestamp). +- end: the end date of the report, likewise. +- agentnum: the agent to limit the report to, if any. + +=cut + +sub queue_liability_report { + my $job = shift; + my $param = shift; + + my $cgi = new CGI; + $cgi->param('beginning', $param->{beginning}); + $cgi->param('ending', $param->{ending}); + my($beginning, $ending) = FS::UI::Web::parse_beginning_ending($cgi); + my $agentnum = $param->{agentnum}; + if ($agentnum =~ /^(\d+)$/) { $agentnum = $1; } else { $agentnum = ''; }; + generate_liability_report( + 'beginning' => $beginning, + 'ending' => $ending, + 'agentnum' => $agentnum, + 'p' => $param->{RootURL}, + 'job' => $job, + ); +} - warn "$me upgrading $self\n" if $DEBUG; +=item generate_liability_report PARAMS - my @column = qw ( tax excessrate usetax useexcessrate fee excessfee - feebase feemax ); +Generates a tax liability report. PARAMS must include: - if ( $dbh->{Driver}->{Name} eq 'Pg' ) { +- beginning, as a timestamp +- ending, as a timestamp +- p: the Freeside root URL, for generating links +- agentnum (optional) - eval "use DBI::Const::GetInfoType;"; - die $@ if $@; +=cut - my $major_version = 0; - $dbh->get_info( $GetInfoType{SQL_DBMS_VER} ) =~ /^(\d{2})/ - && ( $major_version = sprintf("%d", $1) ); +#shit, all sorts of false laxiness w/report_newtax.cgi +sub generate_liability_report { + my %args = @_; - if ( $major_version > 7 ) { + my ( $count, $last, $min_sec ) = _progressbar_foo(); - # ideally this would be supported in DBIx-DBSchema and friends + #let us open the temp file early + my $dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc; + my $report = new File::Temp( TEMPLATE => 'report.tax.liability.XXXXXXXX', + DIR => $dir, + UNLINK => 0, # not so temp + ) or die "can't open report file: $!\n"; - foreach my $column ( @column ) { - my $columndef = dbdef->table($self->table)->column($column); - unless ($columndef->type eq 'numeric') { + my $conf = new FS::Conf; + my $money_char = $conf->config('money_char') || '$'; - warn "updating tax_rate column $column to numeric\n" if $DEBUG; - my $sql = "ALTER TABLE tax_rate ALTER $column TYPE numeric(14,8)"; - my $sth = $dbh->prepare($sql) or die $dbh->errstr; - $sth->execute or die $sth->errstr; + my $join_cust = " + JOIN cust_bill USING ( invnum ) + LEFT JOIN cust_main USING ( custnum ) + "; - warn "updating h_tax_rate column $column to numeric\n" if $DEBUG; - $sql = "ALTER TABLE h_tax_rate ALTER $column TYPE numeric(14,8)"; - $sth = $dbh->prepare($sql) or die $dbh->errstr; - $sth->execute or die $sth->errstr; + my $join_loc = + "LEFT JOIN cust_bill_pkg_tax_rate_location USING ( billpkgnum )"; + my $join_tax_loc = "LEFT JOIN tax_rate_location USING ( taxratelocationnum )"; - } + my $addl_from = " $join_cust $join_loc $join_tax_loc "; + + my $where = "WHERE _date >= $args{beginning} AND _date <= $args{ending} "; + + my $agentname = ''; + if ( $args{agentnum} =~ /^(\d+)$/ ) { + my $agent = qsearchs('agent', { 'agentnum' => $1 } ); + die "agent not found" unless $agent; + $agentname = $agent->agent; + $where .= ' AND cust_main.agentnum = '. $agent->agentnum; + } + + #my @taxparam = ( 'itemdesc', 'tax_rate_location.state', 'tax_rate_location.county', 'tax_rate_location.city', 'cust_bill_pkg_tax_rate_location.locationtaxid' ); + my @taxparams = qw( city county state locationtaxid ); + my @params = ('itemdesc', @taxparams); + + my $select = 'DISTINCT itemdesc,locationtaxid,tax_rate_location.state,tax_rate_location.county,tax_rate_location.city'; + + #false laziness w/FS::Report::Table::Monthly (sub should probably be moved up + #to FS::Report or FS::Record or who the fuck knows where) + my $scalar_sql = sub { + my( $r, $param, $sql ) = @_; + my $sth = dbh->prepare($sql) or die dbh->errstr; + $sth->execute( map $r->$_(), @$param ) + or die "Unexpected error executing statement $sql: ". $sth->errstr; + $sth->fetchrow_arrayref->[0] || 0; + }; + + my $tax = 0; + my $credit = 0; + my %taxes = (); + my %basetaxes = (); + my $calculated = 0; + + # get all distinct tuples of (tax name, state, county, city, locationtaxid) + # for taxes that have been charged + # (state, county, city are from tax_rate_location, not from customer data) + my @tax_and_location = qsearch({ table => 'cust_bill_pkg', + select => $select, + hashref => { pkgpart => 0 }, + addl_from => $addl_from, + extra_sql => $where, + debug => 1, + }); + $count = scalar(@tax_and_location); + foreach my $t ( @tax_and_location ) { + + if ( $args{job} ) { + if ( time - $min_sec > $last ) { + $args{job}->update_statustext( int( 100 * $calculated / $count ). + ",Calculating" + ); + $last = time; } + } - } elsif ( $dbh->{pg_server_version} =~ /^704/ ) { + #my @params = map { my $f = $_; $f =~ s/.*\.//; $f } @taxparam; + my $label = join('~', map { $t->$_ } @params); + $label = 'Tax'. $label if $label =~ /^~/; + unless ( exists( $taxes{$label} ) ) { + my ($baselabel, @trash) = split /~/, $label; - # ideally this would be supported in DBIx-DBSchema and friends + $taxes{$label}->{'label'} = join(', ', split(/~/, $label) ); + $taxes{$label}->{'url_param'} = + join(';', map { "$_=". uri_escape($t->$_) } @params); - foreach my $column ( @column ) { - my $columndef = dbdef->table($self->table)->column($column); - unless ($columndef->type eq 'numeric') { + my $itemdesc_loc = + " ( itemdesc = ? OR ? = '' AND itemdesc IS NULL ) ". + "AND ". FS::tax_rate_location->location_sql( map { $_ => $t->$_ } + @taxparams + ); - warn "updating tax_rate column $column to numeric\n" if $DEBUG; + my $taxwhere = + "FROM cust_bill_pkg $addl_from $where AND $itemdesc_loc"; - foreach my $table ( qw( tax_rate h_tax_rate ) ) { + my $sql = "SELECT SUM(amount) $taxwhere AND cust_bill_pkg.pkgnum = 0"; - my $sql = "ALTER TABLE $table RENAME $column TO old_$column"; - my $sth = $dbh->prepare($sql) or die $dbh->errstr; - $sth->execute or die $sth->errstr; + my $x = &{$scalar_sql}($t, [ 'itemdesc', 'itemdesc' ], $sql ); + $tax += $x; + $taxes{$label}->{'tax'} += $x; - my $def = dbdef->table($table)->column($column); - $def->type('numeric'); - $def->length('14,8'); - my $null = $def->null; - $def->null('NULL'); + my $creditfrom = + "JOIN cust_credit_bill_pkg USING (billpkgnum,billpkgtaxratelocationnum)"; + my $creditwhere = + "FROM cust_bill_pkg $addl_from $creditfrom $where AND $itemdesc_loc"; - $sql = "ALTER TABLE $table ADD COLUMN ". $def->line($dbh); - $sth = $dbh->prepare($sql) or die $dbh->errstr; - $sth->execute or die $sth->errstr; + $sql = "SELECT SUM(cust_credit_bill_pkg.amount) ". + " $creditwhere AND cust_bill_pkg.pkgnum = 0"; - $sql = "UPDATE $table SET $column = CAST( old_$column AS numeric )"; - $sth = $dbh->prepare($sql) or die $dbh->errstr; - $sth->execute or die $sth->errstr; + my $y = &{$scalar_sql}($t, [ 'itemdesc', 'itemdesc' ], $sql ); + $credit += $y; + $taxes{$label}->{'credit'} += $y; - unless ( $null eq 'NULL' ) { - $sql = "ALTER TABLE $table ALTER $column SET NOT NULL"; - $sth = $dbh->prepare($sql) or die $dbh->errstr; - $sth->execute or die $sth->errstr; - } + unless ( exists( $taxes{$baselabel} ) ) { - $sql = "ALTER TABLE $table DROP old_$column"; - $sth = $dbh->prepare($sql) or die $dbh->errstr; - $sth->execute or die $sth->errstr; + $basetaxes{$baselabel}->{'label'} = $baselabel; + $basetaxes{$baselabel}->{'url_param'} = "itemdesc=$baselabel"; + $basetaxes{$baselabel}->{'base'} = 1; - } - } } - } else { + $basetaxes{$baselabel}->{'tax'} += $x; + $basetaxes{$baselabel}->{'credit'} += $y; + + } - warn "WARNING: tax_rate table upgrade unsupported for this Pg version\n"; + # calculate customer-exemption for this tax + # calculate package-exemption for this tax + # calculate monthly exemption (texas tax) for this tax + # count up all the cust_tax_exempt_pkg records associated with + # the actual line items. + } - } - } else { + #ordering + + if ( $args{job} ) { + $args{job}->update_statustext( "0,Sorted" ); + $last = time; + } - warn "WARNING: tax_rate table upgrade only supported for Pg 8+\n"; + my @taxes = (); + foreach my $tax ( sort { $a cmp $b } keys %taxes ) { + my ($base, @trash) = split '~', $tax; + my $basetax = delete( $basetaxes{$base} ); + if ($basetax) { + if ( $basetax->{tax} == $taxes{$tax}->{tax} ) { + $taxes{$tax}->{base} = 1; + } else { + push @taxes, $basetax; + } + } + push @taxes, $taxes{$tax}; } - ''; + push @taxes, { + 'label' => 'Total', + 'url_param' => '', + 'tax' => $tax, + 'credit' => $credit, + 'base' => 1, + }; + + + my $dateagentlink = "begin=$args{beginning};end=$args{ending}"; + $dateagentlink .= ';agentnum='. $args{agentnum} + if length($agentname); + my $baselink = $args{p}. "search/cust_bill_pkg.cgi?vendortax=1;" . + $dateagentlink; + my $creditlink = $args{p}. "search/cust_credit_bill_pkg.html?$dateagentlink"; + + print $report < + + <% include('/elements/table-grid.html') %> + + + + + Tax invoiced +      + + Tax credited + +EOF + + my $bgcolor1 = '#eeeeee'; + my $bgcolor2 = '#ffffff'; + my $bgcolor = ''; + + $count = scalar(@taxes); + $calculated = 0; + foreach my $tax ( @taxes ) { + + if ( $args{job} ) { + if ( time - $min_sec > $last ) { + $args{job}->update_statustext( int( 100 * $calculated / $count ). + ",Generated" + ); + $last = time; + } + } + + if ( $bgcolor eq $bgcolor1 ) { + $bgcolor = $bgcolor2; + } else { + $bgcolor = $bgcolor1; + } + + my $link = ''; + if ( $tax->{'label'} ne 'Total' ) { + $link = ';'. $tax->{'url_param'}; + } + + print $report < + <% '$tax->{label}' %> + <% ($tax->{base}) ? qq!! : '' %> + + <% '$money_char' %><% sprintf('%.2f', $tax->{'tax'} ) %> + + <% !($tax->{base}) ? qq!! : '' %> + + <% ($tax->{base}) ? qq!! : '' %> + + <% '$money_char' %><% sprintf('%.2f', $tax->{'credit'} ) %> + + <% !($tax->{base}) ? qq!! : '' %> + +EOF + } + + print $report < + + + +EOF + + my $reportname = $report->filename; + close $report; + + my $dropstring = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc. '/report.'; + $reportname =~ s/^$dropstring//; + + my $reporturl = "%%%ROOTURL%%%/misc/queued_report.html?report=$reportname"; + die "view\n"; } + + =back =head1 BUGS + Highly specific to CCH taxes. This should arguably go in some kind of + subclass (FS::tax_rate::CCH) with auto-reblessing, similar to part_pkg + subclasses. But currently there aren't any other options, so. + Mixing automatic and manual editing works poorly at present. + Tax liability calculations take too long and arguably don't belong here. + Tax liability report generation not entirely safe (escaped). + + Sparse documentation. + =head1 SEE ALSO -L, L, L, schema.html from the base -documentation. +L, L, L =cut