use strict;
use vars qw( @EXPORT_OK $conf
- @cust_main_county %cust_main_county $countyflag ); # $cityflag );
+ @cust_main_county %cust_main_county $countyflag $DEBUG $me); # $cityflag );
use Exporter;
use FS::Record qw( qsearch qsearchs dbh );
use FS::cust_bill_pkg;
use FS::part_pkg;
use FS::cust_tax_exempt;
use FS::cust_tax_exempt_pkg;
+use FS::upgrade_journal;
+
+$DEBUG = 0;
+$me = '[FS::cust_main_county]';
@EXPORT_OK = qw( regionselector );
=item recurtax - if 'Y', this tax does not apply to recurring fees
+=item source - the tax lookup method that created this tax record. For records
+created manually, this will be null.
+
=back
=head1 METHODS
sub check {
my $self = shift;
+ $self->trim_whitespace(qw(district city county state country));
+ $self->set('city', uc($self->get('city'))); # also county?
+
$self->exempt_amount(0) unless $self->exempt_amount;
$self->ut_numbern('taxnum')
|| $self->ut_textn('taxname')
|| $self->ut_enum('setuptax', [ '', 'Y' ] )
|| $self->ut_enum('recurtax', [ '', 'Y' ] )
+ || $self->ut_textn('source')
|| $self->SUPER::check
;
}
+sub _merge_into {
+ # for internal use: takes another cust_main_county object, transfers
+ # all existing references to this record to that one, and deletes this
+ # one.
+ my $record = shift;
+ my $other = shift or die "record to merge into must be provided";
+ my $new_taxnum = $other->taxnum;
+ my $old_taxnum = $record->taxnum;
+ if ($other->tax != $record->tax or
+ $other->exempt_amount != $record->exempt_amount) {
+ # don't assume these are the same.
+ warn "Found duplicate taxes (#$new_taxnum and #$old_taxnum) but they have different rates and can't be merged.\n";
+ } else {
+ warn "Merging tax #$old_taxnum into #$new_taxnum\n";
+ foreach my $table (qw(
+ cust_bill_pkg_tax_location
+ cust_bill_pkg_tax_location_void
+ cust_tax_exempt_pkg
+ cust_tax_exempt_pkg_void
+ )) {
+ foreach my $row (qsearch($table, { 'taxnum' => $old_taxnum })) {
+ $row->set('taxnum' => $new_taxnum);
+ my $error = $row->replace;
+ die $error if $error;
+ }
+ }
+ my $error = $record->delete;
+ die $error if $error;
+ }
+}
+
+=item process_edit_import
+
+=cut
+
+use Data::Dumper;
+sub process_edit_import {
+ my $job = shift;
+
+ my $opt = { 'table' => 'cust_main_county',
+ 'params' => [], #required, apparantly
+ 'formats' => { 'default' => [
+ 'country',
+ 'state',
+ 'county',
+ 'city',
+ '', #tax class
+ 'taxname',
+ 'tax',
+ 'old_tax', #old tax
+ ] },
+ 'format_headers' => { 'default' => 1, },
+ 'format_types' => { 'default' => 'xls' },
+ };
+
+ #false laziness w/
+ #FS::Record::process_batch_import( $job, $opt, @_ );
+
+ my $table = $opt->{table};
+ my @pass_params = @{ $opt->{params} };
+ my %formats = %{ $opt->{formats} };
+
+ my $param = shift;
+ warn Dumper($param) if $DEBUG;
+
+ my $files = $param->{'uploaded_files'}
+ or die "No files provided.\n";
+
+ my (%files) = map { /^(\w+):([\.\w]+)$/ ? ($1,$2):() } split /,/, $files;
+
+ my $dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc. '/';
+ my $file = $dir. $files{'file'};
+
+ my $error =
+ #false laziness w/
+ #FS::Record::batch_import( {
+ FS::cust_main_county::edit_import( {
+ #class-static
+ table => $table,
+ formats => \%formats,
+ format_types => $opt->{format_types},
+ format_headers => $opt->{format_headers},
+ format_sep_chars => $opt->{format_sep_chars},
+ format_fixedlength_formats => $opt->{format_fixedlength_formats},
+ #per-import
+ job => $job,
+ file => $file,
+ #type => $type,
+ format => $param->{format},
+ params => { map { $_ => $param->{$_} } @pass_params },
+ #?
+ default_csv => $opt->{default_csv},
+ } );
+
+ unlink $file;
+
+ die "$error\n" if $error;
+
+}
+
+=item edit_import
+
+=cut
+
+#false laziness w/ #FS::Record::batch_import, grep "edit_import" for differences
+#could be turned into callbacks or something
+use Text::CSV_XS;
+sub edit_import {
+ my $param = shift;
+
+ warn "$me edit_import call with params: \n". Dumper($param)
+ if $DEBUG;
+
+ my $table = $param->{table};
+ my $formats = $param->{formats};
+
+ my $job = $param->{job};
+ my $file = $param->{file};
+ my $format = $param->{'format'};
+ my $params = $param->{params} || {};
+
+ die "unknown format $format" unless exists $formats->{ $format };
+
+ my $type = $param->{'format_types'}
+ ? $param->{'format_types'}{ $format }
+ : $param->{type} || 'csv';
+
+ unless ( $type ) {
+ if ( $file =~ /\.(\w+)$/i ) {
+ $type = lc($1);
+ } else {
+ #or error out???
+ warn "can't parse file type from filename $file; defaulting to CSV";
+ $type = 'csv';
+ }
+ $type = 'csv'
+ if $param->{'default_csv'} && $type ne 'xls';
+ }
+
+ my $header = $param->{'format_headers'}
+ ? $param->{'format_headers'}{ $param->{'format'} }
+ : 0;
+
+ my $sep_char = $param->{'format_sep_chars'}
+ ? $param->{'format_sep_chars'}{ $param->{'format'} }
+ : ',';
+
+ my $fixedlength_format =
+ $param->{'format_fixedlength_formats'}
+ ? $param->{'format_fixedlength_formats'}{ $param->{'format'} }
+ : '';
+
+ my @fields = @{ $formats->{ $format } };
+
+ my $row = 0;
+ my $count;
+ my $parser;
+ my @buffer = ();
+ my @header = (); #edit_import
+ if ( $type eq 'csv' || $type eq 'fixedlength' ) {
+
+ if ( $type eq 'csv' ) {
+
+ my %attr = ();
+ $attr{sep_char} = $sep_char if $sep_char;
+ $parser = new Text::CSV_XS \%attr;
+
+ } elsif ( $type eq 'fixedlength' ) {
+
+ eval "use Parse::FixedLength;";
+ die $@ if $@;
+ $parser = new Parse::FixedLength $fixedlength_format;
+
+ } else {
+ die "Unknown file type $type\n";
+ }
+
+ @buffer = split(/\r?\n/, slurp($file) );
+ splice(@buffer, 0, ($header || 0) );
+ $count = scalar(@buffer);
+
+ } elsif ( $type eq 'xls' ) {
+
+ eval "use Spreadsheet::ParseExcel;";
+ die $@ if $@;
+
+ eval "use DateTime::Format::Excel;";
+ #for now, just let the error be thrown if it is used, since only CDR
+ # formats bill_west and troop use it, not other excel-parsing things
+ #die $@ if $@;
+
+ my $excel = Spreadsheet::ParseExcel::Workbook->new->Parse($file);
+
+ $parser = $excel->{Worksheet}[0]; #first sheet
+
+ $count = $parser->{MaxRow} || $parser->{MinRow};
+ $count++;
+
+ $row = $header || 0;
+
+ #edit_import - need some magic to parse the header
+ if ( $header ) {
+ my @header_row = @{ $parser->{Cells}[$0] };
+ @header = map $_->{Val}, @header_row;
+ }
+
+ } else {
+ die "Unknown file type $type\n";
+ }
+
+ #my $columns;
+
+ local $SIG{HUP} = 'IGNORE';
+ local $SIG{INT} = 'IGNORE';
+ local $SIG{QUIT} = 'IGNORE';
+ local $SIG{TERM} = 'IGNORE';
+ local $SIG{TSTP} = 'IGNORE';
+ local $SIG{PIPE} = 'IGNORE';
+
+ my $oldAutoCommit = $FS::UID::AutoCommit;
+ local $FS::UID::AutoCommit = 0;
+ my $dbh = dbh;
+
+ my $line;
+ my $imported = 0;
+ my( $last, $min_sec ) = ( time, 5 ); #progressbar foo
+ while (1) {
+
+ my @columns = ();
+ if ( $type eq 'csv' ) {
+
+ last unless scalar(@buffer);
+ $line = shift(@buffer);
+
+ $parser->parse($line) or do {
+ $dbh->rollback if $oldAutoCommit;
+ return "can't parse: ". $parser->error_input();
+ };
+ @columns = $parser->fields();
+
+ } elsif ( $type eq 'fixedlength' ) {
+
+ @columns = $parser->parse($line);
+
+ } elsif ( $type eq 'xls' ) {
+
+ last if $row > ($parser->{MaxRow} || $parser->{MinRow})
+ || ! $parser->{Cells}[$row];
+
+ my @row = @{ $parser->{Cells}[$row] };
+ @columns = map $_->{Val}, @row;
+
+ #my $z = 'A';
+ #warn $z++. ": $_\n" for @columns;
+
+ } else {
+ die "Unknown file type $type\n";
+ }
+
+ #edit_import loop
+
+ my %hash = %$params;
+ my @later;
+
+ foreach my $field ( @fields ) {
+
+ my $value = shift @columns;
+
+ if ( ref($field) eq 'CODE' ) {
+ #&{$field}(\%hash, $value);
+ push @later, $field, $value;
+ } elsif ($field) { #edit_import
+ $hash{$field} = $value if defined($value) && length($value);
+ }
+
+ }
+
+ my $class = "FS::$table";
+
+ my $record = $class->new( \%hash );
+
+ while ( scalar(@later) ) {
+ my $sub = shift @later;
+ my $data = shift @later;
+ &{$sub}($record, $data); #edit_import - don't have $conf
+ }
+
+ #edit_import update or insert, not just insert
+ my $old = qsearchs({
+ 'table' => $table,
+ 'hashref' => { map { $_ => $record->$_() } qw(country state county city taxname) },
+ });
+
+ my $error;
+ if ( $old ) {
+ $record->taxnum($old->taxnum);
+ $error = $record->replace($old)
+ } else {
+ $record->insert;
+ }
+
+ if ( $error ) {
+ $dbh->rollback if $oldAutoCommit;
+ return "can't insert record". ( $line ? " for $line" : '' ). ": $error";
+ }
+
+ $row++;
+ $imported++;
+
+ if ( $job && time - $min_sec > $last ) { #progress bar
+ $job->update_statustext( int(100 * $imported / $count) );
+ $last = time;
+ }
+
+ }
+
+ $dbh->commit or die $dbh->errstr if $oldAutoCommit;;
+
+ return "Empty file!" unless $imported || $param->{empty_ok};
+
+ ''; #no error
+
+}
+
+sub _upgrade_data {
+ my $class = shift;
+ # assume taxes in Washington with district numbers, and null name, or
+ # named 'sales tax', are looked up via the wa_sales method. mark them.
+ my $journal = 'cust_main_county__source_wa_sales_201611';
+ if (!FS::upgrade_journal->is_done($journal)) {
+ my @taxes = qsearch({
+ 'table' => 'cust_main_county',
+ 'extra_sql' => " WHERE tax > 0 AND country = 'US' AND state = 'WA'".
+ " AND district IS NOT NULL AND ( taxname IS NULL OR ".
+ " taxname ~* 'sales tax' )",
+ });
+ if ( @taxes ) {
+ warn "Flagging Washington state sales taxes: ".scalar(@taxes)." records.\n";
+ foreach (@taxes) {
+ $_->set('source', 'wa_sales');
+ my $error = $_->replace;
+ die $error if $error;
+ }
+ }
+ FS::upgrade_journal->set_done($journal);
+ }
+ my @key_fields = (qw(city county state country district taxname taxclass));
+
+ # trim whitespace and convert to uppercase in the 'city' field.
+ foreach my $record (qsearch({
+ table => 'cust_main_county',
+ extra_sql => " WHERE city LIKE ' %' OR city LIKE '% ' OR city != UPPER(city)",
+ })) {
+ # any with-trailing-space records probably duplicate other records
+ # from the same city, and if we just fix the record in place, we'll
+ # create an exact duplicate.
+ # so find the record this one would duplicate, and merge them.
+ $record->check; # trims whitespace
+ my %match = map { $_ => $record->get($_) } @key_fields;
+ my $other = qsearchs('cust_main_county', \%match);
+ if ($other) {
+ $record->_merge_into($other);
+ } else {
+ # else there is no record this one duplicates, so just fix it
+ my $error = $record->replace;
+ die $error if $error;
+ }
+ } # foreach $record
+
+ # separate wa_sales taxes by tax class as needed
+ my $district_taxname = $conf->config('tax_district_taxname');
+ $journal = 'cust_main_county__district_taxclass';
+ if (!FS::upgrade_journal->is_done($journal)
+ and $conf->exists('enable_taxclasses')) {
+ eval "use FS::part_pkg_taxclass";
+ my @taxes = qsearch({
+ 'table' => 'cust_main_county',
+ 'extra_sql' => " WHERE tax > 0 AND country = 'US' AND state = 'WA'".
+ " AND district IS NOT NULL AND source = 'wa_sales'".
+ " AND taxclass IS NULL"
+ });
+ my @classes = FS::part_pkg_taxclass->taxclass_names;
+ if ( @taxes ) {
+ warn "Separating WA sales taxes: ".scalar(@taxes)." records.\n";
+ foreach my $oldtax (@taxes) {
+ my $error;
+ my $taxnum = $oldtax->taxnum;
+ warn "Separating tax #$taxnum into classes\n";
+ foreach my $taxclass (@classes) {
+ # ensure that we end up with a single copy of the tax in this
+ # jurisdiction+class. there may already be one (or more) there.
+ # if so, they all represent the same tax; merge them together.
+ my %newtax_hash = (
+ 'country' => 'US',
+ 'state' => 'WA',
+ 'city' => $oldtax->city,
+ 'district' => $oldtax->district,
+ 'taxclass' => $taxclass,
+ 'source' => 'wa_sales',
+ );
+ my @taxes_in_class = qsearch('cust_main_county', {
+ %newtax_hash,
+ 'tax' => { op => '>', value => 0 },
+ 'setuptax' => '',
+ 'recurtax' => '',
+ });
+ my $newtax = shift @taxes_in_class;
+ if ($newtax) {
+ foreach (@taxes_in_class) {
+ # allow the merge, even if this somehow differs.
+ $_->set('tax', $newtax->tax);
+ $_->_merge_into($newtax);
+ }
+ }
+ $newtax ||= FS::cust_main_county->new(\%newtax_hash);
+ # copy properties from the pre-split tax
+ $newtax->set('tax', $oldtax->tax);
+ $newtax->set('setuptax', $oldtax->setuptax);
+ $newtax->set('recurtax', $oldtax->recurtax);
+ # and assign the defined tax name
+ $newtax->set('taxname', $district_taxname);
+ $error = ($newtax->taxnum ? $newtax->replace : $newtax->insert);
+ die "splitting taxnum ".$oldtax->taxnum.": $error\n" if $error;
+ } # foreach $taxclass
+ $oldtax->set('tax', 0);
+ $error = $oldtax->replace;
+ die "splitting taxnum ".$oldtax->taxnum.": $error\n" if $error;
+ }
+ }
+ FS::upgrade_journal->set_done($journal);
+ }
+
+ # also ensure they all have the chosen taxname now
+ if ($district_taxname) {
+ my @taxes = qsearch('cust_main_county', {
+ 'source' => 'wa_sales',
+ 'taxname' => { op => '!=', value => $district_taxname }
+ });
+ if (@taxes) {
+ warn "Renaming WA sales taxes: ".scalar(@taxes)." records.\n";
+ foreach my $tax (@taxes) {
+ $tax->set('taxname', $district_taxname);
+ my $error = $tax->replace;
+ die "renaming taxnum ".$tax->taxnum.": $error\n" if $error;
+ }
+ }
+ }
+
+ # remove duplicates (except disabled records)
+ my @duplicate_sets = qsearch({
+ table => 'cust_main_county',
+ select => FS::Record::group_concat_sql('taxnum', ',') . ' AS taxnums, ' .
+ join(',', @key_fields),
+ extra_sql => ' WHERE tax > 0
+ GROUP BY city, county, state, country, district, taxname, taxclass
+ HAVING COUNT(*) > 1'
+ });
+ warn "Found ".scalar(@duplicate_sets)." set(s) of duplicate tax definitions\n"
+ if @duplicate_sets;
+ foreach my $set (@duplicate_sets) {
+ my @taxnums = split(',', $set->get('taxnums'));
+ my $first = FS::cust_main_county->by_key(shift @taxnums);
+ foreach my $taxnum (@taxnums) {
+ my $record = FS::cust_main_county->by_key($taxnum);
+ $record->_merge_into($first);
+ }
+ }
+
+
+ '';
+}
+
=back
=head1 BUGS
projects / freeside.git / blobdiff