+ for my $line (grep $oldlines{$_}, keys %oldlines) {
+
+ $csv->parse($line) or do {
+ #$dbh->rollback if $oldAutoCommit;
+ die "can't parse: ". $csv->error_input();
+ };
+ my @columns = $csv->fields();
+
+ $csv->combine( splice(@columns, 0, $numfields) );
+
+ print $dfh $csv->string, ',"D"', "\n";
+ }
+
+ close $dfh;
+
+ return $diffname;
+}
+
+sub _cch_fetch_and_unzip {
+ my ( $job, $urls, $secret, $dir ) = @_;
+
+ my $ua = new LWP::UserAgent;
+ foreach my $url (split ',', $urls) {
+ my @name = split '/', $url; #somewhat restrictive
+ my $name = pop @name;
+ $name =~ /([\w.]+)/; # untaint that which we don't trust so much any more
+ $name = $1;
+
+ open my $taxfh, ">$dir/$name" or die "Can't open $dir/$name: $!\n";
+
+ my ( $imported, $last, $min_sec ) = _progressbar_foo();
+ my $res = $ua->request(
+ new HTTP::Request( GET => $url ),
+ sub {
+ print $taxfh $_[0] or die "Can't write to $dir/$name: $!\n";
+ my $content_length = $_[1]->content_length;
+ $imported += length($_[0]);
+ if ( time - $min_sec > $last ) {
+ my $error = $job->update_statustext(
+ ($content_length ? int(100 * $imported/$content_length) : 0 ).
+ ",Downloading data from CCH"
+ );
+ die $error if $error;
+ $last = time;
+ }
+ },
+ );
+ die "download of $url failed: ". $res->status_line
+ unless $res->is_success;
+
+ close $taxfh;
+ my $error = $job->update_statustext( "0,Unpacking data" );
+ die $error if $error;
+ $secret =~ /([\w.]+)/; # untaint that which we don't trust so much any more
+ $secret = $1;
+ system('unzip', "-P", $secret, "-d", "$dir", "$dir/$name") == 0
+ or die "unzip -P $secret -d $dir $dir/$name failed";
+ #unlink "$dir/$name";
+ }
+}
+
+sub _cch_extract_csv_from_dbf {
+ my ( $job, $dir, $name ) = @_;
+
+ eval "use XBase;";
+ die $@ if $@;
+
+ my ( $imported, $last, $min_sec ) = _progressbar_foo();
+ my $error = $job->update_statustext( "0,Unpacking $name" );
+ die $error if $error;
+ warn "opening $dir.new/$name.dbf\n" if $DEBUG;
+ my $table = new XBase 'name' => "$dir.new/$name.dbf";
+ die "failed to access $dir.new/$name.dbf: ". XBase->errstr
+ unless defined($table);
+ my $count = $table->last_record; # approximately;
+ open my $csvfh, ">$dir.new/$name.txt"
+ or die "failed to open $dir.new/$name.txt: $!\n";
+
+ my $csv = new Text::CSV_XS { 'always_quote' => 1 };
+ my @fields = $table->field_names;
+ my $cursor = $table->prepare_select;
+ my $format_date =
+ sub { my $date = shift;
+ $date =~ /^(\d{4})(\d{2})(\d{2})$/ && ($date = "$2/$3/$1");
+ $date;
+ };
+ while (my $row = $cursor->fetch_hashref) {
+ $csv->combine( map { my $type = $table->field_type($_);
+ if ($type eq 'D') {
+ &{$format_date}($row->{$_}) ;
+ } elsif ($type eq 'N' && $row->{$_} =~ /e-/i ) {
+ sprintf('%.8f', $row->{$_}); #db row is numeric(14,8)
+ } else {
+ $row->{$_};
+ }
+ }
+ @fields
+ );
+ print $csvfh $csv->string, "\n";
+ $imported++;
+ if ( time - $min_sec > $last ) {
+ my $error = $job->update_statustext(
+ int(100 * $imported/$count). ",Unpacking $name"
+ );
+ die $error if $error;
+ $last = time;
+ }
+ }
+ $table->close;
+ close $csvfh;
+}
+
+sub _remember_disabled_taxes {
+ my ( $job, $format, $disabled_tax_rate ) = @_;
+
+ # cch specific hash
+
+ my ( $imported, $last, $min_sec ) = _progressbar_foo();
+
+ my @items = qsearch( { table => 'tax_rate',
+ hashref => { disabled => 'Y',
+ data_vendor => $format,
+ },
+ select => 'geocode, taxclassnum',
+ }
+ );
+ my $count = scalar(@items);
+ foreach my $tax_rate ( @items ) {
+ if ( time - $min_sec > $last ) {
+ $job->update_statustext(
+ int( 100 * $imported / $count ). ",Remembering disabled taxes"
+ );
+ $last = time;
+ }
+ $imported++;
+ my $tax_class =
+ qsearchs( 'tax_class', { taxclassnum => $tax_rate->taxclassnum } );
+ unless ( $tax_class ) {
+ warn "failed to find tax_class ". $tax_rate->taxclassnum;
+ next;
+ }
+ $disabled_tax_rate->{$tax_rate->geocode. ':'. $tax_class->taxclass} = 1;
+ }
+}
+
+sub _remember_tax_products {
+ my ( $job, $format, $taxproduct ) = @_;
+
+ # XXX FIXME this loop only works when cch is the only data provider
+
+ my ( $imported, $last, $min_sec ) = _progressbar_foo();
+
+ my $extra_sql = "
+ WHERE taxproductnum IS NOT NULL
+ OR EXISTS ( SELECT 1 from part_pkg_option
+ WHERE part_pkg_option.pkgpart = part_pkg.pkgpart
+ AND optionname LIKE 'usage_taxproductnum_%'
+ AND optionvalue != ''
+ )
+ ";
+ my @items = qsearch( { table => 'part_pkg',
+ select => 'DISTINCT pkgpart,taxproductnum',
+ hashref => {},
+ extra_sql => $extra_sql,
+ }
+ );
+ my $count = scalar(@items);
+ foreach my $part_pkg ( @items ) {
+ if ( time - $min_sec > $last ) {
+ $job->update_statustext(
+ int( 100 * $imported / $count ). ",Remembering tax products"
+ );
+ $last = time;
+ }
+ $imported++;
+ warn "working with package part ". $part_pkg->pkgpart.
+ "which has a taxproductnum of ". $part_pkg->taxproductnum. "\n" if $DEBUG;
+ my $part_pkg_taxproduct = $part_pkg->taxproduct('');
+ $taxproduct->{$part_pkg->pkgpart}->{''} = $part_pkg_taxproduct->taxproduct
+ if $part_pkg_taxproduct && $part_pkg_taxproduct->data_vendor eq $format;
+
+ foreach my $option ( $part_pkg->part_pkg_option ) {
+ next unless $option->optionname =~ /^usage_taxproductnum_(\w+)$/;
+ my $class = $1;
+
+ $part_pkg_taxproduct = $part_pkg->taxproduct($class);
+ $taxproduct->{$part_pkg->pkgpart}->{$class} =
+ $part_pkg_taxproduct->taxproduct
+ if $part_pkg_taxproduct && $part_pkg_taxproduct->data_vendor eq $format;
+ }
+ }
+}
+
+sub _restore_remembered_tax_products {
+ my ( $job, $format, $taxproduct ) = @_;
+
+ # cch specific
+
+ my ( $imported, $last, $min_sec ) = _progressbar_foo();
+ my $count = scalar(keys %$taxproduct);
+ foreach my $pkgpart ( keys %$taxproduct ) {
+ warn "restoring taxproductnums on pkgpart $pkgpart\n" if $DEBUG;
+ if ( time - $min_sec > $last ) {
+ $job->update_statustext(
+ int( 100 * $imported / $count ). ",Restoring tax products"
+ );
+ $last = time;
+ }
+ $imported++;
+
+ my $part_pkg = qsearchs('part_pkg', { pkgpart => $pkgpart } );
+ unless ( $part_pkg ) {
+ return "somehow failed to find part_pkg with pkgpart $pkgpart!\n";
+ }
+
+ my %options = $part_pkg->options;
+ my %pkg_svc = map { $_->svcpart => $_->quantity } $part_pkg->pkg_svc;
+ my $primary_svc = $part_pkg->svcpart;
+ my $new = new FS::part_pkg { $part_pkg->hash };
+
+ foreach my $class ( keys %{ $taxproduct->{$pkgpart} } ) {
+ warn "working with class '$class'\n" if $DEBUG;
+ my $part_pkg_taxproduct =
+ qsearchs( 'part_pkg_taxproduct',
+ { taxproduct => $taxproduct->{$pkgpart}->{$class},
+ data_vendor => $format,
+ }
+ );
+
+ unless ( $part_pkg_taxproduct ) {
+ return "failed to find part_pkg_taxproduct (".
+ $taxproduct->{$pkgpart}->{$class}. ") for pkgpart $pkgpart\n";