X-Git-Url: http://git.freeside.biz/gitweb/?a=blobdiff_plain;f=FS%2FFS%2Fcust_main.pm;h=718fccf2f1de56373e7104cf78402c0f4ea3b5da;hb=0269c850cfefc00d5da255f88c63a314e1ab6cd0;hp=ff441dbc0fce597ab703a1059330b071670f89cb;hpb=931aa447df899ead7e5dd6ded2243d5288f8df6f;p=freeside.git

diff --git a/FS/FS/cust_main.pm b/FS/FS/cust_main.pm
index ff441dbc0..718fccf2f 100644
--- a/FS/FS/cust_main.pm
+++ b/FS/FS/cust_main.pm
@@ -8,6 +8,7 @@ use vars qw( $realtime_bop_decline_quiet ); #ugh
 use Safe;
 use Carp;
 use Exporter;
+use Scalar::Util qw( blessed );
 use Time::Local qw(timelocal_nocheck);
 use Data::Dumper;
 use Tie::IxHash;
@@ -15,13 +16,14 @@ use Digest::MD5 qw(md5_base64);
 use Date::Format;
 use Date::Parse;
 #use Date::Manip;
+use File::Slurp qw( slurp );
+use File::Temp qw( tempfile );
 use String::Approx qw(amatch);
 use Business::CreditCard 0.28;
 use Locale::Country;
-use Data::Dumper;
 use FS::UID qw( getotaker dbh driver_name );
 use FS::Record qw( qsearchs qsearch dbdef );
-use FS::Misc qw( send_email generate_ps do_print );
+use FS::Misc qw( generate_email send_email generate_ps do_print );
 use FS::Msgcat qw(gettext);
 use FS::cust_pkg;
 use FS::cust_svc;
@@ -35,6 +37,7 @@ use FS::cust_credit;
 use FS::cust_refund;
 use FS::part_referral;
 use FS::cust_main_county;
+use FS::cust_tax_location;
 use FS::agent;
 use FS::cust_main_invoice;
 use FS::cust_credit_bill;
@@ -45,8 +48,6 @@ use FS::part_pkg;
 use FS::part_event;
 use FS::part_event_condition;
 #use FS::cust_event;
-use FS::cust_tax_exempt;
-use FS::cust_tax_exempt_pkg;
 use FS::type_pkgs;
 use FS::payment_gateway;
 use FS::agent_payment_gateway;
@@ -54,7 +55,7 @@ use FS::banned_pay;
 use FS::payinfo_Mixin;
 use FS::TicketSystem;
 
-@ISA = qw( FS::Record FS::payinfo_Mixin );
+@ISA = qw( FS::payinfo_Mixin FS::Record );
 
 @EXPORT_OK = qw( smart_search );
 
@@ -225,6 +226,8 @@ Card Verification Value, "CVV2" (also known as CVC2 or CID), the 3 or 4 digit nu
 
 =item spool_cdr - Enable individual CDR spooling, empty or `Y'
 
+=item squelch_cdr - Discourage individual CDR printing, empty or `Y'
+
 =back
 
 =head1 METHODS
@@ -1054,7 +1057,7 @@ sub delete {
 
 }
 
-=item replace OLD_RECORD [ INVOICING_LIST_ARYREF ]
+=item replace [ OLD_RECORD ] [ INVOICING_LIST_ARYREF ]
 
 Replaces the OLD_RECORD with this one in the database.  If there is an error,
 returns the error, otherwise returns false.
@@ -1070,23 +1073,16 @@ check_invoicing_list first.  Here's an example:
 
 sub replace {
   my $self = shift;
-  my $old = shift;
+
+  my $old = ( blessed($_[0]) && $_[0]->isa('FS::Record') )
+              ? shift
+              : $self->replace_old;
+
   my @param = @_;
+
   warn "$me replace called\n"
     if $DEBUG;
 
-  local $SIG{HUP} = 'IGNORE';
-  local $SIG{INT} = 'IGNORE';
-  local $SIG{QUIT} = 'IGNORE';
-  local $SIG{TERM} = 'IGNORE';
-  local $SIG{TSTP} = 'IGNORE';
-  local $SIG{PIPE} = 'IGNORE';
-
-  # We absolutely have to have an old vs. new record to make this work.
-  if (!defined($old)) {
-    $old = qsearchs( 'cust_main', { 'custnum' => $self->custnum } );
-  }
-
   my $curuser = $FS::CurrentUser::CurrentUser;
   if (    $self->payby eq 'COMP'
        && $self->payby ne $old->payby
@@ -1101,6 +1097,13 @@ sub replace {
     && $self->payby =~ /^(CARD|DCRD)$/
     && ( $old->payinfo eq $self->payinfo || $old->paymask eq $self->paymask );
 
+  local $SIG{HUP} = 'IGNORE';
+  local $SIG{INT} = 'IGNORE';
+  local $SIG{QUIT} = 'IGNORE';
+  local $SIG{TERM} = 'IGNORE';
+  local $SIG{TSTP} = 'IGNORE';
+  local $SIG{PIPE} = 'IGNORE';
+
   my $oldAutoCommit = $FS::UID::AutoCommit;
   local $FS::UID::AutoCommit = 0;
   my $dbh = dbh;
@@ -1209,6 +1212,7 @@ sub check {
     || $self->ut_number('agentnum')
     || $self->ut_textn('agent_custid')
     || $self->ut_number('refnum')
+    || $self->ut_textn('custbatch')
     || $self->ut_name('last')
     || $self->ut_name('first')
     || $self->ut_snumbern('birthdate')
@@ -1532,7 +1536,7 @@ sub check {
     $self->payname($1);
   }
 
-  foreach my $flag (qw( tax spool_cdr )) {
+  foreach my $flag (qw( tax spool_cdr squelch_cdr )) {
     $self->$flag() =~ /^(Y?)$/ or return "Illegal $flag: ". $self->$flag();
     $self->$flag($1);
   }
@@ -1951,10 +1955,12 @@ sub bill_and_collect {
   # cancel packages
   ###
 
-  #$^T not $options{time} because freeside-daily -d is for pre-printing invoices
-  foreach my $cust_pkg (
-    grep { $_->expire && $_->expire <= $^T } $self->ncancelled_pkgs
-  ) {
+  #$options{actual_time} not $options{time} because freeside-daily -d is for
+  #pre-printing invoices
+  my @cancel_pkgs = grep { $_->expire && $_->expire <= $options{actual_time} }
+                         $self->ncancelled_pkgs;
+
+  foreach my $cust_pkg ( @cancel_pkgs ) {
     my $error = $cust_pkg->cancel;
     warn "Error cancelling expired pkg ". $cust_pkg->pkgnum.
          " for custnum ". $self->custnum. ": $error"
@@ -1965,15 +1971,22 @@ sub bill_and_collect {
   # suspend packages
   ###
 
-  #$^T not $options{time} because freeside-daily -d is for pre-printing invoices
-  foreach my $cust_pkg (
-    grep { (    $_->part_pkg->is_prepaid && $_->bill && $_->bill < $^T
-             || $_->adjourn && $_->adjourn <= $^T
-           )
-           && ! $_->susp
+  #$options{actual_time} not $options{time} because freeside-daily -d is for
+  #pre-printing invoices
+  my @susp_pkgs = 
+    grep { ! $_->susp
+           && (    (    $_->part_pkg->is_prepaid
+                     && $_->bill
+                     && $_->bill < $options{actual_time}
+                   )
+                || (    $_->adjourn
+                    && $_->adjourn <= $options{actual_time}
+                  )
+              )
          }
-         $self->ncancelled_pkgs
-  ) {
+         $self->ncancelled_pkgs;
+
+  foreach my $cust_pkg ( @susp_pkgs ) {
     my $error = $cust_pkg->suspend;
     warn "Error suspending package ". $cust_pkg->pkgnum.
          " for custnum ". $self->custnum. ": $error"
@@ -2034,13 +2047,12 @@ Used in conjunction with the I<time> option, this option specifies the date of f
 sub bill {
   my( $self, %options ) = @_;
   return '' if $self->payby eq 'COMP';
+  local $DEBUG = 1;
   warn "$me bill customer ". $self->custnum. "\n"
     if $DEBUG;
 
   my $time = $options{'time'} || time;
 
-  my $error;
-
   #put below somehow?
   local $SIG{HUP} = 'IGNORE';
   local $SIG{INT} = 'IGNORE';
@@ -2055,34 +2067,22 @@ sub bill {
 
   $self->select_for_update; #mutex
 
-  #create a new invoice
-  #(we'll remove it later if it doesn't actually need to be generated [contains
-  # no line items] and we're inside a transaciton so nothing else will see it)
-  my $cust_bill = new FS::cust_bill ( {
-    'custnum' => $self->custnum,
-    '_date'   => ( $options{'invoice_time'} || $time ),
-    #'charged' => $charged,
-    'charged' => 0,
-  } );
-  $error = $cust_bill->insert;
-  if ( $error ) {
-    $dbh->rollback if $oldAutoCommit;
-    return "can't create invoice for customer #". $self->custnum. ": $error";
-  }
-  my $invnum = $cust_bill->invnum;
+  my @cust_bill_pkg = ();
+  my @appended_cust_bill_pkg = ();
 
   ###
   # find the packages which are due for billing, find out how much they are
   # & generate invoice database.
   ###
 
-  my( $total_setup, $total_recur ) = ( 0, 0 );
+  my( $total_setup, $total_recur, $postal_charge ) = ( 0, 0, 0 );
   my %tax;
+  my %taxlisthash;
+  my %taxname;
   my @precommit_hooks = ();
 
-  foreach my $cust_pkg (
-    qsearch('cust_pkg', { 'custnum' => $self->custnum } )
-  ) {
+  my @cust_pkgs = qsearch('cust_pkg', { 'custnum' => $self->custnum } );
+  foreach my $cust_pkg (@cust_pkgs) {
 
     #NO!! next if $cust_pkg->cancel;  
     next if $cust_pkg->getfield('cancel');  
@@ -2093,351 +2093,186 @@ sub bill {
     $cust_pkg->setfield('bill', '')
       unless defined($cust_pkg->bill);
  
-    my $part_pkg = $cust_pkg->part_pkg;
+    #my $part_pkg = $cust_pkg->part_pkg;
 
+    my $real_pkgpart = $cust_pkg->pkgpart;
     my %hash = $cust_pkg->hash;
-    my $old_cust_pkg = new FS::cust_pkg \%hash;
-
-    my @details = ();
-
-    ###
-    # bill setup
-    ###
-
-    my $setup = 0;
-    if ( ! $cust_pkg->setup &&
-         (
-           ( $conf->exists('disable_setup_suspended_pkgs') &&
-            ! $cust_pkg->getfield('susp')
-          ) || ! $conf->exists('disable_setup_suspended_pkgs')
-         )
-      || $options{'resetup'}
-    ) {
-    
-      warn "    bill setup\n" if $DEBUG > 1;
-
-      $setup = eval { $cust_pkg->calc_setup( $time, \@details ) };
-      if ( $@ ) {
-        $dbh->rollback if $oldAutoCommit;
-        return "$@ running calc_setup for $cust_pkg\n";
-      }
-
-      $cust_pkg->setfield('setup', $time) unless $cust_pkg->setup;
-    }
-
-    ###
-    # bill recurring fee
-    ### 
-
-    my $recur = 0;
-    my $sdate;
-    if ( $part_pkg->getfield('freq') ne '0' &&
-         ! $cust_pkg->getfield('susp') &&
-         ( $cust_pkg->getfield('bill') || 0 ) <= $time
-    ) {
-
-      # XXX should this be a package event?  probably.  events are called
-      # at collection time at the moment, though...
-      if ( $part_pkg->can('reset_usage') ) {
-        warn "    resetting usage counters" if $DEBUG > 1;
-        $part_pkg->reset_usage($cust_pkg);
-      }
 
-      warn "    bill recur\n" if $DEBUG > 1;
+    foreach my $part_pkg ( $cust_pkg->part_pkg->self_and_bill_linked ) {
 
-      # XXX shared with $recur_prog
-      $sdate = $cust_pkg->bill || $cust_pkg->setup || $time;
+      $cust_pkg->set($_, $hash{$_}) foreach qw ( setup last_bill bill );
 
-      #over two params!  lets at least switch to a hashref for the rest...
-      my %param = ( 'precommit_hooks' => \@precommit_hooks, );
-
-      $recur = eval { $cust_pkg->calc_recur( \$sdate, \@details, \%param ) };
-      if ( $@ ) {
-        $dbh->rollback if $oldAutoCommit;
-        return "$@ running calc_recur for $cust_pkg\n";
-      }
-
-      #change this bit to use Date::Manip? CAREFUL with timezones (see
-      # mailing list archive)
-      my ($sec,$min,$hour,$mday,$mon,$year) =
-        (localtime($sdate) )[0,1,2,3,4,5];
-
-      #pro-rating magic - if $recur_prog fiddles $sdate, want to use that
-      # only for figuring next bill date, nothing else, so, reset $sdate again
-      # here
-      $sdate = $cust_pkg->bill || $cust_pkg->setup || $time;
-      $cust_pkg->last_bill($sdate);
-
-      if ( $part_pkg->freq =~ /^\d+$/ ) {
-        $mon += $part_pkg->freq;
-        until ( $mon < 12 ) { $mon -= 12; $year++; }
-      } elsif ( $part_pkg->freq =~ /^(\d+)w$/ ) {
-        my $weeks = $1;
-        $mday += $weeks * 7;
-      } elsif ( $part_pkg->freq =~ /^(\d+)d$/ ) {
-        my $days = $1;
-        $mday += $days;
-      } elsif ( $part_pkg->freq =~ /^(\d+)h$/ ) {
-        my $hours = $1;
-        $hour += $hours;
-      } else {
+      my $error =
+        $self->_make_lines( 'part_pkg'            => $part_pkg,
+                            'cust_pkg'            => $cust_pkg,
+                            'precommit_hooks'     => \@precommit_hooks,
+                            'line_items'          => \@cust_bill_pkg,
+                            'appended_line_items' => \@appended_cust_bill_pkg,
+                            'setup'               => \$total_setup,
+                            'recur'               => \$total_recur,
+                            'tax_matrix'          => \%taxlisthash,
+                            'time'                => $time,
+                            'options'             => \%options,
+                          );
+      if ($error) {
         $dbh->rollback if $oldAutoCommit;
-        return "unparsable frequency: ". $part_pkg->freq;
+        return $error;
       }
-      $cust_pkg->setfield('bill',
-        timelocal_nocheck($sec,$min,$hour,$mday,$mon,$year));
-    }
 
-    warn "\$setup is undefined" unless defined($setup);
-    warn "\$recur is undefined" unless defined($recur);
-    warn "\$cust_pkg->bill is undefined" unless defined($cust_pkg->bill);
+    } #foreach my $part_pkg
 
-    ###
-    # If $cust_pkg has been modified, update it and create cust_bill_pkg records
-    ###
-
-    if ( $cust_pkg->modified ) {  # hmmm.. and if the options are modified?
+  } #foreach my $cust_pkg
 
-      warn "  package ". $cust_pkg->pkgnum. " modified; updating\n"
-        if $DEBUG >1;
+  push @cust_bill_pkg, @appended_cust_bill_pkg;
 
-      $error=$cust_pkg->replace($old_cust_pkg,
-                                options => { $cust_pkg->options },
-                               );
-      if ( $error ) { #just in case
-        $dbh->rollback if $oldAutoCommit;
-        return "Error modifying pkgnum ". $cust_pkg->pkgnum. ": $error";
-      }
+  unless ( @cust_bill_pkg ) { #don't create an invoice w/o line items
+    #but do commit any package date cycling that happened
+    $dbh->commit or die $dbh->errstr if $oldAutoCommit;
+    return '';
+  }
 
-      $setup = sprintf( "%.2f", $setup );
-      $recur = sprintf( "%.2f", $recur );
-      if ( $setup < 0 && ! $conf->exists('allow_negative_charges') ) {
-        $dbh->rollback if $oldAutoCommit;
-        return "negative setup $setup for pkgnum ". $cust_pkg->pkgnum;
-      }
-      if ( $recur < 0 && ! $conf->exists('allow_negative_charges') ) {
+  my $postal_pkg = $self->charge_postal_fee();
+  if ( $postal_pkg && !ref( $postal_pkg ) ) {
+    $dbh->rollback if $oldAutoCommit;
+    return "can't charge postal invoice fee for customer ".
+      $self->custnum. ": $postal_pkg";
+  }
+  if ( $postal_pkg ) {
+    foreach my $part_pkg ( $postal_pkg->part_pkg->self_and_bill_linked ) {
+      my $error =
+        $self->_make_lines( 'part_pkg'            => $part_pkg,
+                            'cust_pkg'            => $postal_pkg,
+                            'precommit_hooks'     => \@precommit_hooks,
+                            'line_items'          => \@cust_bill_pkg,
+                            'appended_line_items' => \@appended_cust_bill_pkg,
+                            'setup'               => \$total_setup,
+                            'recur'               => \$total_recur,
+                            'tax_matrix'          => \%taxlisthash,
+                            'time'                => $time,
+                            'options'             => \%options,
+                          );
+      if ($error) {
         $dbh->rollback if $oldAutoCommit;
-        return "negative recur $recur for pkgnum ". $cust_pkg->pkgnum;
+        return $error;
       }
+    }
+  }
 
-      if ( $setup != 0 || $recur != 0 ) {
-
-        warn "    charges (setup=$setup, recur=$recur); adding line items\n"
-          if $DEBUG > 1;
-        my $cust_bill_pkg = new FS::cust_bill_pkg ({
-          'invnum'  => $invnum,
-          'pkgnum'  => $cust_pkg->pkgnum,
-          'setup'   => $setup,
-          'recur'   => $recur,
-          'sdate'   => $sdate,
-          'edate'   => $cust_pkg->bill,
-          'details' => \@details,
-        });
-        $error = $cust_bill_pkg->insert;
-        if ( $error ) {
-          $dbh->rollback if $oldAutoCommit;
-          return "can't create invoice line item for invoice #$invnum: $error";
-        }
-        $total_setup += $setup;
-        $total_recur += $recur;
-
-        ###
-        # handle taxes
-        ###
-
-        unless ( $self->tax =~ /Y/i || $self->payby eq 'COMP' ) {
-
-          my $prefix = 
-            ( $conf->exists('tax-ship_address') && length($self->ship_last) )
-            ? 'ship_'
-            : '';
-          my %taxhash = map { $_ => $self->get("$prefix$_") }
-                            qw( state county country );
-
-          $taxhash{'taxclass'} = $part_pkg->taxclass;
-
-          my @taxes = qsearch( 'cust_main_county', \%taxhash );
-
-          unless ( @taxes ) {
-            $taxhash{'taxclass'} = '';
-            @taxes =  qsearch( 'cust_main_county', \%taxhash );
-          }
-
-          #one more try at a whole-country tax rate
-          unless ( @taxes ) {
-            $taxhash{$_} = '' foreach qw( state county );
-            @taxes =  qsearch( 'cust_main_county', \%taxhash );
-          }
+  warn "having a look at the taxes we found...\n" if $DEBUG > 2;
+  foreach my $tax ( keys %taxlisthash ) {
+    my $tax_object = shift @{ $taxlisthash{$tax} };
+    warn "found ". $tax_object->taxname. " as $tax\n" if $DEBUG > 2;
+    my $listref_or_error = $tax_object->taxline( @{ $taxlisthash{$tax} } );
+    unless (ref($listref_or_error)) {
+      $dbh->rollback if $oldAutoCommit;
+      return $listref_or_error;
+    }
+    unshift @{ $taxlisthash{$tax} }, $tax_object;
 
-          # maybe eliminate this entirely, along with all the 0% records
-          unless ( @taxes ) {
-            $dbh->rollback if $oldAutoCommit;
-            return
-              "fatal: can't find tax rate for state/county/country/taxclass ".
-              join('/', ( map $self->get("$prefix$_"),
-                              qw(state county country)
-                        ),
-                        $part_pkg->taxclass ). "\n";
-          }
+    warn "adding ". $listref_or_error->[1].
+         " as ". $listref_or_error->[0]. "\n"
+      if $DEBUG > 2;
+    $tax{ $tax_object->taxname } += $listref_or_error->[1];
+    if ( $taxname{ $listref_or_error->[0] } ) {
+      push @{ $taxname{ $listref_or_error->[0] } }, $tax_object->taxname;
+    }else{
+      $taxname{ $listref_or_error->[0] } = [ $tax_object->taxname ];
+    }
   
-          foreach my $tax ( @taxes ) {
-
-            my $taxable_charged = 0;
-            $taxable_charged += $setup
-              unless $part_pkg->setuptax =~ /^Y$/i
-                  || $tax->setuptax =~ /^Y$/i;
-            $taxable_charged += $recur
-              unless $part_pkg->recurtax =~ /^Y$/i
-                  || $tax->recurtax =~ /^Y$/i;
-            next unless $taxable_charged;
-
-            if ( $tax->exempt_amount && $tax->exempt_amount > 0 ) {
-              #my ($mon,$year) = (localtime($sdate) )[4,5];
-              my ($mon,$year) = (localtime( $sdate || $cust_bill->_date ) )[4,5];
-              $mon++;
-              my $freq = $part_pkg->freq || 1;
-              if ( $freq !~ /(\d+)$/ ) {
-                $dbh->rollback if $oldAutoCommit;
-                return "daily/weekly package definitions not (yet?)".
-                       " compatible with monthly tax exemptions";
-              }
-              my $taxable_per_month =
-                sprintf("%.2f", $taxable_charged / $freq );
-
-              #call the whole thing off if this customer has any old
-              #exemption records...
-              my @cust_tax_exempt =
-                qsearch( 'cust_tax_exempt' => { custnum=> $self->custnum } );
-              if ( @cust_tax_exempt ) {
-                $dbh->rollback if $oldAutoCommit;
-                return
-                  'this customer still has old-style tax exemption records; '.
-                  'run bin/fs-migrate-cust_tax_exempt?';
-              }
+  }
 
-              foreach my $which_month ( 1 .. $freq ) {
-
-                #maintain the new exemption table now
-                my $sql = "
-                  SELECT SUM(amount)
-                    FROM cust_tax_exempt_pkg
-                      LEFT JOIN cust_bill_pkg USING ( billpkgnum )
-                      LEFT JOIN cust_bill     USING ( invnum     )
-                    WHERE custnum = ?
-                      AND taxnum  = ?
-                      AND year    = ?
-                      AND month   = ?
-                ";
-                my $sth = dbh->prepare($sql) or do {
-                  $dbh->rollback if $oldAutoCommit;
-                  return "fatal: can't lookup exising exemption: ". dbh->errstr;
-                };
-                $sth->execute(
-                  $self->custnum,
-                  $tax->taxnum,
-                  1900+$year,
-                  $mon,
-                ) or do {
-                  $dbh->rollback if $oldAutoCommit;
-                  return "fatal: can't lookup exising exemption: ". dbh->errstr;
-                };
-                my $existing_exemption = $sth->fetchrow_arrayref->[0] || 0;
-                
-                my $remaining_exemption =
-                  $tax->exempt_amount - $existing_exemption;
-                if ( $remaining_exemption > 0 ) {
-                  my $addl = $remaining_exemption > $taxable_per_month
-                    ? $taxable_per_month
-                    : $remaining_exemption;
-                  $taxable_charged -= $addl;
-
-                  my $cust_tax_exempt_pkg = new FS::cust_tax_exempt_pkg ( {
-                    'billpkgnum' => $cust_bill_pkg->billpkgnum,
-                    'taxnum'     => $tax->taxnum,
-                    'year'       => 1900+$year,
-                    'month'      => $mon,
-                    'amount'     => sprintf("%.2f", $addl ),
-                  } );
-                  $error = $cust_tax_exempt_pkg->insert;
-                  if ( $error ) {
-                    $dbh->rollback if $oldAutoCommit;
-                    return "fatal: can't insert cust_tax_exempt_pkg: $error";
-                  }
-                } # if $remaining_exemption > 0
-
-                #++
-                $mon++;
-                #until ( $mon < 12 ) { $mon -= 12; $year++; }
-                until ( $mon < 13 ) { $mon -= 12; $year++; }
-  
-              } #foreach $which_month
+  #some taxes are taxed
+  my %totlisthash;
   
-            } #if $tax->exempt_amount
-
-            $taxable_charged = sprintf( "%.2f", $taxable_charged);
-
-            #$tax += $taxable_charged * $cust_main_county->tax / 100
-            $tax{ $tax->taxname || 'Tax' } +=
-              $taxable_charged * $tax->tax / 100
-
-          } #foreach my $tax ( @taxes )
-
-        } #unless $self->tax =~ /Y/i || $self->payby eq 'COMP'
-
-      } #if $setup != 0 || $recur != 0
-      
-    } #if $cust_pkg->modified
+  warn "finding taxed taxes...\n" if $DEBUG > 2;
+  foreach my $tax ( keys %taxlisthash ) {
+    my $tax_object = shift @{ $taxlisthash{$tax} };
+    warn "found possible taxed tax ". $tax_object->taxname. " we call $tax\n"
+      if $DEBUG > 2;
+    next unless $tax_object->can('tax_on_tax');
 
-  } #foreach my $cust_pkg
+    foreach my $tot ( $tax_object->tax_on_tax( $self ) ) {
+      my $totname = ref( $tot ). ' '. $tot->taxnum;
 
-  unless ( $cust_bill->cust_bill_pkg ) {
-    $cust_bill->delete; #don't create an invoice w/o line items
+      warn "checking $totname which we call ". $tot->taxname. " as applicable\n"
+        if $DEBUG > 2;
+      next unless exists( $taxlisthash{ $totname } ); # only increase
+                                                      # existing taxes
+      warn "adding $totname to taxed taxes\n" if $DEBUG > 2;
+      if ( exists( $totlisthash{ $totname } ) ) {
+        push @{ $totlisthash{ $totname  } }, $tax{ $tax_object->taxname };
+      }else{
+        $totlisthash{ $totname } = [ $tot, $tax{ $tax_object->taxname } ];
+      }
+    }
+  }
 
-   # XXX this seems to be broken
-   #( DBD::Pg::st execute failed: ERROR:  syntax error at or near "hcb" )
-#   # get rid of our fake history too, waste of unecessary space
-#    my $h_cleanup_query = q{
-#      DELETE FROM h_cust_bill hcb
-#       WHERE hcb.invnum = ?
-#      AND NOT EXISTS ( SELECT 1 FROM cust_bill cb where cb.invnum = hcb.invnum )
-#    };
-#    my $h_sth = $dbh->prepare($h_cleanup_query);
-#    $h_sth->execute($invnum);
+  warn "having a look at taxed taxes...\n" if $DEBUG > 2;
+  foreach my $tax ( keys %totlisthash ) {
+    my $tax_object = shift @{ $totlisthash{$tax} };
+    warn "found previously found taxed tax ". $tax_object->taxname. "\n"
+      if $DEBUG > 2;
+    my $listref_or_error = $tax_object->taxline( @{ $totlisthash{$tax} } );
+    unless (ref($listref_or_error)) {
+      $dbh->rollback if $oldAutoCommit;
+      return $listref_or_error;
+    }
 
-    $dbh->commit or die $dbh->errstr if $oldAutoCommit;
-    return '';
+    warn "adding taxed tax amount ". $listref_or_error->[1].
+         " as ". $tax_object->taxname. "\n"
+      if $DEBUG;
+    $tax{ $tax_object->taxname } += $listref_or_error->[1];
   }
+  
+  #consolidate and create tax line items
+  warn "consolidating and generating...\n" if $DEBUG > 2;
+  foreach my $taxname ( keys %taxname ) {
+    my $tax = 0;
+    my %seen = ();
+    warn "adding $taxname\n" if $DEBUG > 1;
+    foreach my $taxitem ( @{ $taxname{$taxname} } ) {
+      $tax += $tax{$taxitem} unless $seen{$taxitem};
+      warn "adding $tax{$taxitem}\n" if $DEBUG > 1;
+    }
+    next unless $tax;
 
-  my $charged = sprintf( "%.2f", $total_setup + $total_recur );
-
-  foreach my $taxname ( grep { $tax{$_} > 0 } keys %tax ) {
-    my $tax = sprintf("%.2f", $tax{$taxname} );
-    $charged = sprintf( "%.2f", $charged+$tax );
+    $tax = sprintf('%.2f', $tax );
+    $total_setup = sprintf('%.2f', $total_setup+$tax );
   
-    my $cust_bill_pkg = new FS::cust_bill_pkg ({
-      'invnum'   => $invnum,
+    push @cust_bill_pkg, new FS::cust_bill_pkg {
       'pkgnum'   => 0,
       'setup'    => $tax,
       'recur'    => 0,
       'sdate'    => '',
       'edate'    => '',
       'itemdesc' => $taxname,
-    });
-    $error = $cust_bill_pkg->insert;
-    if ( $error ) {
-      $dbh->rollback if $oldAutoCommit;
-      return "can't create invoice line item for invoice #$invnum: $error";
-    }
-    $total_setup += $tax;
+    };
 
   }
 
-  $cust_bill->charged( sprintf( "%.2f", $total_setup + $total_recur ) );
-  $error = $cust_bill->replace;
+  my $charged = sprintf('%.2f', $total_setup + $total_recur );
+
+  #create the new invoice
+  my $cust_bill = new FS::cust_bill ( {
+    'custnum' => $self->custnum,
+    '_date'   => ( $options{'invoice_time'} || $time ),
+    'charged' => $charged,
+  } );
+  my $error = $cust_bill->insert;
   if ( $error ) {
     $dbh->rollback if $oldAutoCommit;
-    return "can't update charged for invoice #$invnum: $error";
+    return "can't create invoice for customer #". $self->custnum. ": $error";
+  }
+
+  foreach my $cust_bill_pkg ( @cust_bill_pkg ) {
+    $cust_bill_pkg->invnum($cust_bill->invnum); 
+    my $error = $cust_bill_pkg->insert;
+    if ( $error ) {
+      $dbh->rollback if $oldAutoCommit;
+      return "can't create invoice line item: $error";
+    }
   }
+    
 
   foreach my $hook ( @precommit_hooks ) { 
     eval {
@@ -2453,134 +2288,566 @@ sub bill {
   ''; #no error
 }
 
-=item collect OPTIONS
 
-(Attempt to) collect money for this customer's outstanding invoices (see
-L<FS::cust_bill>).  Usually used after the bill method.
+sub _make_lines {
+  my ($self, %params) = @_;
 
-Actions are now triggered by billing events; see L<FS::part_event> and the
-billing events web interface.  Old-style invoice events (see
-L<FS::part_bill_event>) have been deprecated.
+  my $part_pkg = $params{part_pkg} or die "no part_pkg specified";
+  my $cust_pkg = $params{cust_pkg} or die "no cust_pkg specified";
+  my $precommit_hooks = $params{precommit_hooks} or die "no package specified";
+  my $cust_bill_pkgs = $params{line_items} or die "no line buffer specified";
+  my $appended_cust_bill_pkg = $params{appended_line_items}
+    or die "no appended line buffer specified";
+  my $total_setup = $params{setup} or die "no setup accumulator specified";
+  my $total_recur = $params{recur} or die "no recur accumulator specified";
+  my $taxlisthash = $params{tax_matrix} or die "no tax accumulator specified";
+  my $time = $params{'time'} or die "no time specified";
+  my (%options) = %{$params{options}};  #hmmm  only for 'resetup'
 
-If there is an error, returns the error, otherwise returns false.
+  my $dbh = dbh;
+  my $real_pkgpart = $cust_pkg->pkgpart;
+  my %hash = $cust_pkg->hash;
+  my $old_cust_pkg = new FS::cust_pkg \%hash;
 
-Options are passed as name-value pairs.
+  my @details = ();
 
-Currently available options are:
+  my $lineitems = 0;
 
-=over 4
+  $cust_pkg->pkgpart($part_pkg->pkgpart);
 
-=item invoice_time
+  ###
+  # bill setup
+  ###
 
-Use this time when deciding when to print invoices and late notices on those invoices.  The default is now.  It is specified as a UNIX timestamp; see L<perlfunc/"time">).  Also see L<Time::Local> and L<Date::Parse> for conversion functions.
+  my $setup = 0;
+  my $unitsetup = 0;
+  if ( ! $cust_pkg->setup &&
+       (
+         ( $conf->exists('disable_setup_suspended_pkgs') &&
+          ! $cust_pkg->getfield('susp')
+        ) || ! $conf->exists('disable_setup_suspended_pkgs')
+       )
+    || $options{'resetup'}
+  ) {
+    
+    warn "    bill setup\n" if $DEBUG > 1;
+    $lineitems++;
 
-=item retry
+    $setup = eval { $cust_pkg->calc_setup( $time, \@details ) };
+    return "$@ running calc_setup for $cust_pkg\n"
+      if $@;
 
-Retry card/echeck/LEC transactions even when not scheduled by invoice events.
+    $unitsetup = $cust_pkg->part_pkg->unit_setup || $setup; #XXX uuh
 
-=item quiet
+    $cust_pkg->setfield('setup', $time)
+      unless $cust_pkg->setup;
+          #do need it, but it won't get written to the db
+          #|| $cust_pkg->pkgpart != $real_pkgpart;
 
-set true to surpress email card/ACH decline notices.
+  }
 
-=item check_freq
+  ###
+  # bill recurring fee
+  ### 
+
+  #XXX unit stuff here too
+  my $recur = 0;
+  my $unitrecur = 0;
+  my $sdate;
+  if ( $part_pkg->getfield('freq') ne '0' &&
+       ! $cust_pkg->getfield('susp') &&
+       ( $cust_pkg->getfield('bill') || 0 ) <= $time
+  ) {
 
-"1d" for the traditional, daily events (the default), or "1m" for the new monthly events (part_event.check_freq)
+    # XXX should this be a package event?  probably.  events are called
+    # at collection time at the moment, though...
+    $part_pkg->reset_usage($cust_pkg, 'debug'=>$DEBUG)
+      if $part_pkg->can('reset_usage');
+      #don't want to reset usage just cause we want a line item??
+      #&& $part_pkg->pkgpart == $real_pkgpart;
 
-=item payby
+    warn "    bill recur\n" if $DEBUG > 1;
+    $lineitems++;
 
-allows for one time override of normal customer billing method
+    # XXX shared with $recur_prog
+    $sdate = $cust_pkg->bill || $cust_pkg->setup || $time;
 
-=item debug
+    #over two params!  lets at least switch to a hashref for the rest...
+    my %param = ( 'precommit_hooks' => $precommit_hooks, );
 
-Debugging level.  Default is 0 (no debugging), or can be set to 1 (passed-in options), 2 (traces progress), 3 (more information), or 4 (include full search queries)
+    $recur = eval { $cust_pkg->calc_recur( \$sdate, \@details, \%param ) };
+    return "$@ running calc_recur for $cust_pkg\n"
+      if ( $@ );
 
+  
+    #change this bit to use Date::Manip? CAREFUL with timezones (see
+    # mailing list archive)
+    my ($sec,$min,$hour,$mday,$mon,$year) =
+      (localtime($sdate) )[0,1,2,3,4,5];
+    
+    #pro-rating magic - if $recur_prog fiddles $sdate, want to use that
+    # only for figuring next bill date, nothing else, so, reset $sdate again
+    # here
+    $sdate = $cust_pkg->bill || $cust_pkg->setup || $time;
+    $cust_pkg->last_bill($sdate);
+    
+    if ( $part_pkg->freq =~ /^\d+$/ ) {
+      $mon += $part_pkg->freq;
+      until ( $mon < 12 ) { $mon -= 12; $year++; }
+    } elsif ( $part_pkg->freq =~ /^(\d+)w$/ ) {
+      my $weeks = $1;
+      $mday += $weeks * 7;
+    } elsif ( $part_pkg->freq =~ /^(\d+)d$/ ) {
+      my $days = $1;
+      $mday += $days;
+    } elsif ( $part_pkg->freq =~ /^(\d+)h$/ ) {
+      my $hours = $1;
+      $hour += $hours;
+    } else {
+      return "unparsable frequency: ". $part_pkg->freq;
+    }
+    $cust_pkg->setfield('bill',
+      timelocal_nocheck($sec,$min,$hour,$mday,$mon,$year));
 
-=back
+  }
 
-=cut
+  warn "\$setup is undefined" unless defined($setup);
+  warn "\$recur is undefined" unless defined($recur);
+  warn "\$cust_pkg->bill is undefined" unless defined($cust_pkg->bill);
+  
+  ###
+  # If there's line items, create em cust_bill_pkg records
+  # If $cust_pkg has been modified, update it (if we're a real pkgpart)
+  ###
 
-sub collect {
-  my( $self, %options ) = @_;
-  my $invoice_time = $options{'invoice_time'} || time;
+  if ( $lineitems ) {
 
-  #put below somehow?
-  local $SIG{HUP} = 'IGNORE';
-  local $SIG{INT} = 'IGNORE';
-  local $SIG{QUIT} = 'IGNORE';
-  local $SIG{TERM} = 'IGNORE';
-  local $SIG{TSTP} = 'IGNORE';
-  local $SIG{PIPE} = 'IGNORE';
+    if ( $cust_pkg->modified && $cust_pkg->pkgpart == $real_pkgpart ) {
+      # hmm.. and if just the options are modified in some weird price plan?
+  
+      warn "  package ". $cust_pkg->pkgnum. " modified; updating\n"
+        if $DEBUG >1;
+  
+      my $error = $cust_pkg->replace( $old_cust_pkg,
+                                      'options' => { $cust_pkg->options },
+                                    );
+      return "Error modifying pkgnum ". $cust_pkg->pkgnum. ": $error"
+        if $error; #just in case
+    }
+  
+    $setup = sprintf( "%.2f", $setup );
+    $recur = sprintf( "%.2f", $recur );
+    if ( $setup < 0 && ! $conf->exists('allow_negative_charges') ) {
+      return "negative setup $setup for pkgnum ". $cust_pkg->pkgnum;
+    }
+    if ( $recur < 0 && ! $conf->exists('allow_negative_charges') ) {
+      return "negative recur $recur for pkgnum ". $cust_pkg->pkgnum;
+    }
 
-  my $oldAutoCommit = $FS::UID::AutoCommit;
-  local $FS::UID::AutoCommit = 0;
-  my $dbh = dbh;
+    if ( $setup != 0 || $recur != 0 ) {
 
-  $self->select_for_update; #mutex
+      warn "    charges (setup=$setup, recur=$recur); adding line items\n"
+        if $DEBUG > 1;
+      my $cust_bill_pkg = new FS::cust_bill_pkg {
+        'pkgnum'    => $cust_pkg->pkgnum,
+        'setup'     => $setup,
+        'unitsetup' => $unitsetup,
+        'recur'     => $recur,
+        'unitrecur' => $unitrecur,
+        'quantity'  => $cust_pkg->quantity,
+        'sdate'     => $sdate,
+        'edate'     => $cust_pkg->bill,
+        'details'   => \@details,
+      };
+      $cust_bill_pkg->pkgpart_override($part_pkg->pkgpart)
+        unless $part_pkg->pkgpart == $real_pkgpart;
 
-  if ( $DEBUG ) {
-    my $balance = $self->balance;
-    warn "$me collect customer ". $self->custnum. ": balance $balance\n"
-  }
+      $$total_setup += $setup;
+      $$total_recur += $recur;
 
-  if ( exists($options{'retry_card'}) ) {
-    carp 'retry_card option passed to collect is deprecated; use retry';
-    $options{'retry'} ||= $options{'retry_card'};
-  }
-  if ( exists($options{'retry'}) && $options{'retry'} ) {
-    my $error = $self->retry_realtime;
-    if ( $error ) {
-      $dbh->rollback if $oldAutoCommit;
-      return $error;
-    }
-  }
+      ###
+      # handle taxes
+      ###
 
-  # false laziness w/pay_batch::import_results
+      unless ( $self->tax =~ /Y/i || $self->payby eq 'COMP' ) {
 
-  my $due_cust_event = $self->due_cust_event(
-    'debug'      => ( $options{'debug'} || 0 ),
-    'time'       => $invoice_time,
-    'check_freq' => $options{'check_freq'},
-  );
-  unless( ref($due_cust_event) ) {
-    $dbh->rollback if $oldAutoCommit;
-    return $due_cust_event;
-  }
+        #some garbage disappears on cust_bill_pkg refactor
+        my $err_or_cust_bill_pkg =
+          $self->_handle_taxes($part_pkg, $taxlisthash, $cust_bill_pkg);
 
-  foreach my $cust_event ( @$due_cust_event ) {
+        return $err_or_cust_bill_pkg
+          unless ( ref($err_or_cust_bill_pkg) );
 
-    #XXX lock event
-    
-    #re-eval event conditions (a previous event could have changed things)
-    unless ( $cust_event->test_conditions( 'time' => $invoice_time ) ) {
-      #don't leave stray "new/locked" records around
-      my $error = $cust_event->delete;
-      if ( $error ) {
-        #gah, even with transactions
-        $dbh->commit if $oldAutoCommit; #well.
-        return $error;
-      }
-      next;
-    }
+        push @$cust_bill_pkgs, @$err_or_cust_bill_pkg;
 
-    {
-      local $realtime_bop_decline_quiet = 1 if $options{'quiet'};
-      warn "  running cust_event ". $cust_event->eventnum. "\n"
-        if $DEBUG > 1;
+      } #unless $self->tax =~ /Y/i || $self->payby eq 'COMP'
 
+    } #if $setup != 0 || $recur != 0
       
-      #if ( my $error = $cust_event->do_event(%options) ) { #XXX %options?
-      if ( my $error = $cust_event->do_event() ) {
-        #XXX wtf is this?  figure out a proper dealio with return value
-        #from do_event
-	  # gah, even with transactions.
-	  $dbh->commit if $oldAutoCommit; #well.
-	  return $error;
-	}
-    }
+  } #if $line_items
 
-  }
+  if ( $part_pkg->can('append_cust_bill_pkgs') ) {
+    my %param = ( 'precommit_hooks' => $precommit_hooks, );
+    my ($more_cust_bill_pkgs) =
+      eval { $part_pkg->append_cust_bill_pkgs( $cust_pkg, \$sdate, \%param ) };
 
-  $dbh->commit or die $dbh->errstr if $oldAutoCommit;
+    return "$@ running append_cust_bill_pkgs for $cust_pkg\n"
+      if ( $@ );
+    return "$more_cust_bill_pkgs"
+      unless ( ref($more_cust_bill_pkgs) );
+
+    foreach my $cust_bill_pkg ( @{$more_cust_bill_pkgs} ) {
+
+      $cust_bill_pkg->pkgpart_override($part_pkg->pkgpart)
+        unless $part_pkg->pkgpart == $real_pkgpart;
+
+      unless ($cust_bill_pkg->duplicate) {
+        $$total_setup += $cust_bill_pkg->setup;
+        $$total_recur += $cust_bill_pkg->recur;
+
+        ###
+        # handle taxes
+        ###
+
+        unless ( $self->tax =~ /Y/i || $self->payby eq 'COMP' ) {
+
+          #some garbage disappears on cust_bill_pkg refactor
+          my $err_or_cust_bill_pkg =
+            $self->_handle_taxes($part_pkg, $taxlisthash, $cust_bill_pkg);
+
+          return $err_or_cust_bill_pkg
+            unless ( ref($err_or_cust_bill_pkg) );
+
+          push @$appended_cust_bill_pkg, @$err_or_cust_bill_pkg;
+
+        } #unless $self->tax =~ /Y/i || $self->payby eq 'COMP'
+      }
+    }
+  }
+
+}
+
+sub _handle_taxes {
+  my $self = shift;
+  my $part_pkg = shift;
+  my $taxlisthash = shift;
+  my $cust_bill_pkg = shift;
+
+  my %cust_bill_pkg = ();
+  my %taxes = ();
+    
+  my $prefix = 
+    ( $conf->exists('tax-ship_address') && length($self->ship_last) )
+    ? 'ship_'
+    : '';
+
+  my @classes;
+  #push @classes, $cust_bill_pkg->usage_classes if $cust_bill_pkg->type eq 'U';
+  push @classes, $cust_bill_pkg->usage_classes if $cust_bill_pkg->usage;
+  push @classes, 'setup' if $cust_bill_pkg->setup;
+  push @classes, 'recur' if $cust_bill_pkg->recur;
+
+  if ( $conf->exists('enable_taxproducts')
+       && (scalar($part_pkg->part_pkg_taxoverride) || $part_pkg->has_taxproduct)
+     )
+  { 
+
+    foreach my $class (@classes) {
+      my $err_or_ref = $self->_gather_taxes( $part_pkg, $class, $prefix );
+      return $err_or_ref unless ref($err_or_ref);
+      $taxes{$class} = $err_or_ref;
+    }
+
+    unless (exists $taxes{''}) {
+      my $err_or_ref = $self->_gather_taxes( $part_pkg, '', $prefix );
+      return $err_or_ref unless ref($err_or_ref);
+      $taxes{''} = $err_or_ref;
+    }
+
+  }else{
+
+    my %taxhash = map { $_ => $self->get("$prefix$_") }
+                      qw( state county country );
+
+    $taxhash{'taxclass'} = $part_pkg->taxclass;
+
+    my @taxes = qsearch( 'cust_main_county', \%taxhash );
+
+    unless ( @taxes ) {
+      $taxhash{'taxclass'} = '';
+      @taxes =  qsearch( 'cust_main_county', \%taxhash );
+    }
+
+    #one more try at a whole-country tax rate
+    unless ( @taxes ) {
+      $taxhash{$_} = '' foreach qw( state county );
+      @taxes =  qsearch( 'cust_main_county', \%taxhash );
+    }
+
+    $taxes{''} = [ @taxes ];
+    $taxes{'setup'} = [ @taxes ];
+    $taxes{'recur'} = [ @taxes ];
+    $taxes{$_} = [ @taxes ] foreach (@classes);
+
+    # maybe eliminate this entirely, along with all the 0% records
+    unless ( @taxes ) {
+      return
+        "fatal: can't find tax rate for state/county/country/taxclass ".
+        join('/', ( map $self->get("$prefix$_"),
+                        qw(state county country)
+                  ),
+                  $part_pkg->taxclass ). "\n";
+    }
+
+  } #if $conf->exists('enable_taxproducts') 
+
+  # XXX all this goes away with cust_bill_pay refactor
+
+  $cust_bill_pkg{setup} = $cust_bill_pkg if $cust_bill_pkg->setup;
+  $cust_bill_pkg{recur} = $cust_bill_pkg if $cust_bill_pkg->recur;
+    
+  #split setup and recur
+  if ($cust_bill_pkg->setup && $cust_bill_pkg->recur) {
+    my $cust_bill_pkg_recur = new FS::cust_bill_pkg { $cust_bill_pkg->hash };
+    $cust_bill_pkg->set('details', []);
+    $cust_bill_pkg->recur(0);
+    $cust_bill_pkg->unitrecur(0);
+    $cust_bill_pkg->type('');
+    $cust_bill_pkg_recur->setup(0);
+    $cust_bill_pkg_recur->unitsetup(0);
+    $cust_bill_pkg{recur} = $cust_bill_pkg_recur;
+  }
+
+  #split usage from recur
+  my $usage = sprintf( "%.2f", $cust_bill_pkg{recur}->usage );
+  warn "usage is $usage\n" if $DEBUG;
+  if ($usage) {
+    my $cust_bill_pkg_usage =
+        new FS::cust_bill_pkg { $cust_bill_pkg{recur}->hash };
+    $cust_bill_pkg_usage->recur( $usage );
+    $cust_bill_pkg_usage->type( 'U' );
+    my $recur = sprintf( "%.2f", $cust_bill_pkg{recur}->recur - $usage );
+    $cust_bill_pkg{recur}->recur( $recur );
+    $cust_bill_pkg{recur}->type( '' );
+    $cust_bill_pkg{recur}->set('details', []);
+    $cust_bill_pkg{''} = $cust_bill_pkg_usage;
+  }
+
+  #subdivide usage by usage_class
+  if (exists($cust_bill_pkg{''})) {
+    foreach my $class (grep {$_ && $_ ne 'setup' && $_ ne 'recur' } @classes) {
+      my $usage = sprintf( "%.2f", $cust_bill_pkg{''}->usage($class) );
+      my $cust_bill_pkg_usage =
+          new FS::cust_bill_pkg { $cust_bill_pkg{''}->hash };
+      $cust_bill_pkg_usage->recur( $usage );
+      $cust_bill_pkg_usage->set('details', []);
+      my $classless = sprintf( "%.2f", $cust_bill_pkg{''}->recur - $usage );
+      $cust_bill_pkg{''}->recur( $classless );
+      $cust_bill_pkg{$class} = $cust_bill_pkg_usage;
+    }
+    delete $cust_bill_pkg{''} unless $cust_bill_pkg{''}->recur;
+  }
+
+  foreach my $key (keys %cust_bill_pkg) {
+    my @taxes = @{ $taxes{$key} };
+    my $cust_bill_pkg = $cust_bill_pkg{$key};
+
+    foreach my $tax ( @taxes ) {
+      my $taxname = ref( $tax ). ' '. $tax->taxnum;
+      if ( exists( $taxlisthash->{ $taxname } ) ) {
+        push @{ $taxlisthash->{ $taxname  } }, $cust_bill_pkg;
+      }else{
+        $taxlisthash->{ $taxname } = [ $tax, $cust_bill_pkg ];
+      }
+    }
+  }
+
+  # sort setup,recur,'', and the rest numeric && return
+  my @result = map { $cust_bill_pkg{$_} }
+               sort { my $ad = ($a=~/^\d+$/); my $bd = ($b=~/^\d+$/);
+                      ( $ad cmp $bd ) || ( $ad ? $a<=>$b : $b cmp $a )
+                    }
+               keys %cust_bill_pkg;
+
+  \@result;
+}
+
+sub _gather_taxes {
+  my $self = shift;
+  my $part_pkg = shift;
+  my $class = shift;
+  my $prefix = shift;
+
+  my @taxes = ();
+  my $geocode = $self->geocode('cch');
+
+  my @taxclassnums = map { $_->taxclassnum }
+                     $part_pkg->part_pkg_taxoverride($class);
+
+  unless (@taxclassnums) {
+    @taxclassnums = map { $_->taxclassnum }
+                    $part_pkg->part_pkg_taxrate('cch', $geocode, $class);
+  }
+  warn "Found taxclassnum values of ". join(',', @taxclassnums)
+    if $DEBUG;
+
+  my $extra_sql =
+    "AND (".
+    join(' OR ', map { "taxclassnum = $_" } @taxclassnums ). ")";
+
+  @taxes = qsearch({ 'table' => 'tax_rate',
+                     'hashref' => { 'geocode' => $geocode, },
+                     'extra_sql' => $extra_sql,
+                  })
+    if scalar(@taxclassnums);
+
+  # maybe eliminate this entirely, along with all the 0% records
+  unless ( @taxes ) {
+    return 
+      "fatal: can't find tax rate for zip/taxproduct/pkgpart ".
+      join('/', ( map $self->get("$prefix$_"),
+                      qw(zip)
+                ),
+                $part_pkg->taxproduct_description,
+                $part_pkg->pkgpart ). "\n";
+  }
+
+  warn "Found taxes ".
+       join(',', map{ ref($_). " ". $_->get($_->primary_key) } @taxes). "\n" 
+   if $DEBUG;
+
+  [ @taxes ];
+
+}
+
+=item collect OPTIONS
+
+(Attempt to) collect money for this customer's outstanding invoices (see
+L<FS::cust_bill>).  Usually used after the bill method.
+
+Actions are now triggered by billing events; see L<FS::part_event> and the
+billing events web interface.  Old-style invoice events (see
+L<FS::part_bill_event>) have been deprecated.
+
+If there is an error, returns the error, otherwise returns false.
+
+Options are passed as name-value pairs.
+
+Currently available options are:
+
+=over 4
+
+=item invoice_time
+
+Use this time when deciding when to print invoices and late notices on those invoices.  The default is now.  It is specified as a UNIX timestamp; see L<perlfunc/"time">).  Also see L<Time::Local> and L<Date::Parse> for conversion functions.
+
+=item retry
+
+Retry card/echeck/LEC transactions even when not scheduled by invoice events.
+
+=item quiet
+
+set true to surpress email card/ACH decline notices.
+
+=item check_freq
+
+"1d" for the traditional, daily events (the default), or "1m" for the new monthly events (part_event.check_freq)
+
+=item payby
+
+allows for one time override of normal customer billing method
+
+=item debug
+
+Debugging level.  Default is 0 (no debugging), or can be set to 1 (passed-in options), 2 (traces progress), 3 (more information), or 4 (include full search queries)
+
+
+=back
+
+=cut
+
+sub collect {
+  my( $self, %options ) = @_;
+  my $invoice_time = $options{'invoice_time'} || time;
+
+  #put below somehow?
+  local $SIG{HUP} = 'IGNORE';
+  local $SIG{INT} = 'IGNORE';
+  local $SIG{QUIT} = 'IGNORE';
+  local $SIG{TERM} = 'IGNORE';
+  local $SIG{TSTP} = 'IGNORE';
+  local $SIG{PIPE} = 'IGNORE';
+
+  my $oldAutoCommit = $FS::UID::AutoCommit;
+  local $FS::UID::AutoCommit = 0;
+  my $dbh = dbh;
+
+  $self->select_for_update; #mutex
+
+  if ( $DEBUG ) {
+    my $balance = $self->balance;
+    warn "$me collect customer ". $self->custnum. ": balance $balance\n"
+  }
+
+  if ( exists($options{'retry_card'}) ) {
+    carp 'retry_card option passed to collect is deprecated; use retry';
+    $options{'retry'} ||= $options{'retry_card'};
+  }
+  if ( exists($options{'retry'}) && $options{'retry'} ) {
+    my $error = $self->retry_realtime;
+    if ( $error ) {
+      $dbh->rollback if $oldAutoCommit;
+      return $error;
+    }
+  }
+
+  # false laziness w/pay_batch::import_results
+
+  my $due_cust_event = $self->due_cust_event(
+    'debug'      => ( $options{'debug'} || 0 ),
+    'time'       => $invoice_time,
+    'check_freq' => $options{'check_freq'},
+  );
+  unless( ref($due_cust_event) ) {
+    $dbh->rollback if $oldAutoCommit;
+    return $due_cust_event;
+  }
+
+  foreach my $cust_event ( @$due_cust_event ) {
+
+    #XXX lock event
+    
+    #re-eval event conditions (a previous event could have changed things)
+    unless ( $cust_event->test_conditions( 'time' => $invoice_time ) ) {
+      #don't leave stray "new/locked" records around
+      my $error = $cust_event->delete;
+      if ( $error ) {
+        #gah, even with transactions
+        $dbh->commit if $oldAutoCommit; #well.
+        return $error;
+      }
+      next;
+    }
+
+    {
+      local $realtime_bop_decline_quiet = 1 if $options{'quiet'};
+      warn "  running cust_event ". $cust_event->eventnum. "\n"
+        if $DEBUG > 1;
+
+      
+      #if ( my $error = $cust_event->do_event(%options) ) { #XXX %options?
+      if ( my $error = $cust_event->do_event() ) {
+        #XXX wtf is this?  figure out a proper dealio with return value
+        #from do_event
+	  # gah, even with transactions.
+	  $dbh->commit if $oldAutoCommit; #well.
+	  return $error;
+	}
+    }
+
+  }
+
+  $dbh->commit or die $dbh->errstr if $oldAutoCommit;
   '';
 
 }
@@ -3169,7 +3436,7 @@ sub realtime_bop {
     'gatewaynum' => ( $payment_gateway ? $payment_gateway->gatewaynum : '' ),
   };
   $cust_pay_pending->payunique( $options{payunique} )
-    if length($options{payunique});
+    if defined($options{payunique}) && length($options{payunique});
   my $cpp_new_err = $cust_pay_pending->insert; #mutex lost when this is inserted
   return $cpp_new_err if $cpp_new_err;
 
@@ -3322,7 +3589,8 @@ sub realtime_bop {
        'paydate'  => $paydate,
     } );
     #doesn't hurt to know, even though the dup check is in cust_pay_pending now
-    $cust_pay->payunique( $options{payunique} ) if length($options{payunique});
+    $cust_pay->payunique( $options{payunique} )
+      if defined($options{payunique}) && length($options{payunique});
 
     my $oldAutoCommit = $FS::UID::AutoCommit;
     local $FS::UID::AutoCommit = 0;
@@ -4632,16 +4900,22 @@ the error, otherwise returns false.
 
 sub charge {
   my $self = shift;
-  my ( $amount, $pkg, $comment, $taxclass, $additional );
+  my ( $amount, $quantity, $pkg, $comment, $taxclass, $additional, $classnum );
+  my ( $taxproduct, $override );
   if ( ref( $_[0] ) ) {
     $amount     = $_[0]->{amount};
+    $quantity   = exists($_[0]->{quantity}) ? $_[0]->{quantity} : 1;
     $pkg        = exists($_[0]->{pkg}) ? $_[0]->{pkg} : 'One-time charge';
     $comment    = exists($_[0]->{comment}) ? $_[0]->{comment}
                                            : '$'. sprintf("%.2f",$amount);
     $taxclass   = exists($_[0]->{taxclass}) ? $_[0]->{taxclass} : '';
+    $classnum   = exists($_[0]->{classnum}) ? $_[0]->{classnum} : '';
     $additional = $_[0]->{additional};
+    $taxproduct = $_[0]->{taxproductnum};
+    $override   = { '' => $_[0]->{tax_override} };
   }else{
     $amount     = shift;
+    $quantity   = 1;
     $pkg        = @_ ? shift : 'One-time charge';
     $comment    = @_ ? shift : '$'. sprintf("%.2f",$amount);
     $taxclass   = @_ ? shift : '';
@@ -4660,12 +4934,14 @@ sub charge {
   my $dbh = dbh;
 
   my $part_pkg = new FS::part_pkg ( {
-    'pkg'      => $pkg,
-    'comment'  => $comment,
-    'plan'     => 'flat',
-    'freq'     => 0,
-    'disabled' => 'Y',
-    'taxclass' => $taxclass,
+    'pkg'           => $pkg,
+    'comment'       => $comment,
+    'plan'          => 'flat',
+    'freq'          => 0,
+    'disabled'      => 'Y',
+    'classnum'      => $classnum ? $classnum : '',
+    'taxclass'      => $taxclass,
+    'taxproductnum' => $taxproduct,
   } );
 
   my %options = ( ( map { ("additional_info$_" => $additional->[$_] ) }
@@ -4675,7 +4951,9 @@ sub charge {
                   'setup_fee' => $amount,
                 );
 
-  my $error = $part_pkg->insert( options => \%options );
+  my $error = $part_pkg->insert( options       => \%options,
+                                 tax_overrides => $override,
+                               );
   if ( $error ) {
     $dbh->rollback if $oldAutoCommit;
     return $error;
@@ -4693,8 +4971,9 @@ sub charge {
   }
 
   my $cust_pkg = new FS::cust_pkg ( {
-    'custnum' => $self->custnum,
-    'pkgpart' => $pkgpart,
+    'custnum'  => $self->custnum,
+    'pkgpart'  => $pkgpart,
+    'quantity' => $quantity,
   } );
 
   $error = $cust_pkg->insert;
@@ -4708,6 +4987,33 @@ sub charge {
 
 }
 
+#=item charge_postal_fee
+#
+#Applies a one time charge this customer.  If there is an error,
+#returns the error, returns the cust_pkg charge object or false
+#if there was no charge.
+#
+#=cut
+#
+# This should be a customer event.  For that to work requires that bill
+# also be a customer event.
+
+sub charge_postal_fee {
+  my $self = shift;
+
+  my $pkgpart = $conf->config('postal_invoice-fee_pkgpart');
+  return '' unless ($pkgpart && grep { $_ eq 'POST' } $self->invoicing_list);
+
+  my $cust_pkg = new FS::cust_pkg ( {
+    'custnum'  => $self->custnum,
+    'pkgpart'  => $pkgpart,
+    'quantity' => 1,
+  } );
+
+  my $error = $cust_pkg->insert;
+  $error ? $error : $cust_pkg;
+}
+
 =item cust_bill
 
 Returns all the invoices (see L<FS::cust_bill>) for this customer.
@@ -4859,6 +5165,40 @@ sub country_full {
   code2country($self->country);
 }
 
+=item geocode DATA_VENDOR
+
+Returns a value for the customer location as encoded by DATA_VENDOR.
+Currently this only makes sense for "CCH" as DATA_VENDOR.
+
+=cut
+
+sub geocode {
+  my ($self, $data_vendor) = (shift, shift);  #always cch for now
+
+  my $prefix = ( $conf->exists('tax-ship_address') && length($self->ship_last) )
+               ? 'ship_'
+               : '';
+
+  my ($zip,$plus4) = split /-/, $self->get("${prefix}zip")
+    if $self->country eq 'US';
+
+  #CCH specific location stuff
+  my $extra_sql = "AND plus4lo <= '$plus4' AND plus4hi >= '$plus4'";
+
+  my $geocode = '';
+  my $cust_tax_location =
+    qsearchs( {
+                'table'     => 'cust_tax_location', 
+                'hashref'   => { 'zip' => $zip, 'data_vendor' => $data_vendor },
+                'extra_sql' => $extra_sql,
+              }
+            );
+  $geocode = $cust_tax_location->geocode
+    if $cust_tax_location;
+
+  $geocode;
+}
+
 =item cust_status
 
 =item status
@@ -4917,7 +5257,7 @@ Returns a hex triplet color string for this customer's status.
 =cut
 
 use vars qw(%statuscolor);
-tie my %statuscolor, 'Tie::IxHash',
+tie %statuscolor, 'Tie::IxHash',
   'prospect'  => '7e0079', #'000000', #black?  naw, purple
   'active'    => '00CC00', #green
   'inactive'  => '0000CC', #blue
@@ -5021,7 +5361,7 @@ sub prospect_sql { "
 =item active_sql
 
 Returns an SQL expression identifying active cust_main records (customers with
-no active recurring packages, but otherwise unsuspended/uncancelled).
+active recurring packages).
 
 =cut
 
@@ -5033,7 +5373,7 @@ sub active_sql { "
 =item inactive_sql
 
 Returns an SQL expression identifying inactive cust_main records (customers with
-active recurring packages).
+no active recurring packages, but otherwise unsuspended/uncancelled).
 
 =cut
 
@@ -5069,17 +5409,17 @@ sub cancelled_sql { cancel_sql(@_); }
 sub cancel_sql {
 
   my $recurring_sql = FS::cust_pkg->recurring_sql;
-  #my $recurring_sql = "
-  #  '0' != ( select freq from part_pkg
-  #             where cust_pkg.pkgpart = part_pkg.pkgpart )
-  #";
+  my $cancelled_sql = FS::cust_pkg->cancelled_sql;
 
   "
-    0 < ( $select_count_pkgs )
+        0 < ( $select_count_pkgs )
+    AND 0 < ( $select_count_pkgs AND $recurring_sql AND $cancelled_sql   )
     AND 0 = ( $select_count_pkgs AND $recurring_sql
                   AND ( cust_pkg.cancel IS NULL OR cust_pkg.cancel = 0 )
             )
+    AND 0 = (  $select_count_pkgs AND ". FS::cust_pkg->inactive_sql. " )
   ";
+
 }
 
 =item uncancel_sql
@@ -5133,15 +5473,24 @@ Available options are:
 
 =over 4
 
-=item unapplied_date - set to true to disregard unapplied credits, payments and refunds outside the specified time period - by default the time period restriction only applies to invoices (useful for reporting, probably a bad idea for event triggering)
+=item unapplied_date
 
-=item total - set to true to remove all customer comparison clauses, for totals
+set to true to disregard unapplied credits, payments and refunds outside the specified time period - by default the time period restriction only applies to invoices (useful for reporting, probably a bad idea for event triggering)
 
-=item where - WHERE clause hashref (elements "AND"ed together) (typically used with the total option)
+=item total
 
-=item join - JOIN clause (typically used with the total option)
+(unused.  obsolete?)
+set to true to remove all customer comparison clauses, for totals
 
-=item 
+=item where
+
+(unused.  obsolete?)
+WHERE clause hashref (elements "AND"ed together) (typically used with the total option)
+
+=item join
+
+(unused.  obsolete?)
+JOIN clause (typically used with the total option)
 
 =back
 
@@ -5190,8 +5539,8 @@ sub _money_table_where {
   my @where = ();
   push @where, "cust_main.custnum = $table.custnum" unless $opt{'total'};
   if ( $table eq 'cust_bill' || $opt{'unapplied_date'} ) {
-    push @where, "$table._date <= $start" if length($start);
-    push @where, "$table._date >  $end"   if length($end);
+    push @where, "$table._date <= $start" if defined($start) && length($start);
+    push @where, "$table._date >  $end"   if defined($end)   && length($end);
   }
   push @where, @{$opt{'where'}} if $opt{'where'};
   my $where = scalar(@where) ? 'WHERE '. join(' AND ', @where ) : '';
@@ -5200,6 +5549,312 @@ sub _money_table_where {
 
 }
 
+=item search_sql HASHREF
+
+(Class method)
+
+Returns a qsearch hash expression to search for parameters specified in HREF.
+Valid parameters are
+
+=over 4
+
+=item agentnum
+
+=item status
+
+=item cancelled_pkgs
+
+bool
+
+=item signupdate
+
+listref of start date, end date
+
+=item payby
+
+listref
+
+=item current_balance
+
+listref (list returned by FS::UI::Web::parse_lt_gt($cgi, 'current_balance'))
+
+=item cust_fields
+
+=item flattened_pkgs
+
+bool
+
+=back
+
+=cut
+
+sub search_sql {
+  my ($class, $params) = @_;
+
+  my $dbh = dbh;
+
+  my @where = ();
+  my $orderby;
+
+  ##
+  # parse agent
+  ##
+
+  if ( $params->{'agentnum'} =~ /^(\d+)$/ and $1 ) {
+    push @where,
+      "cust_main.agentnum = $1";
+  }
+
+  ##
+  # parse status
+  ##
+
+  #prospect active inactive suspended cancelled
+  if ( grep { $params->{'status'} eq $_ } FS::cust_main->statuses() ) {
+    my $method = $params->{'status'}. '_sql';
+    #push @where, $class->$method();
+    push @where, FS::cust_main->$method();
+  }
+  
+  ##
+  # parse cancelled package checkbox
+  ##
+
+  my $pkgwhere = "";
+
+  $pkgwhere .= "AND (cancel = 0 or cancel is null)"
+    unless $params->{'cancelled_pkgs'};
+
+  ##
+  # dates
+  ##
+
+  foreach my $field (qw( signupdate )) {
+
+    next unless exists($params->{$field});
+
+    my($beginning, $ending) = @{$params->{$field}};
+
+    push @where,
+      "cust_main.$field IS NOT NULL",
+      "cust_main.$field >= $beginning",
+      "cust_main.$field <= $ending";
+
+    $orderby ||= "ORDER BY cust_main.$field";
+
+  }
+
+  ###
+  # payby
+  ###
+
+  my @payby = grep /^([A-Z]{4})$/, @{ $params->{'payby'} };
+  if ( @payby ) {
+    push @where, '( '. join(' OR ', map "cust_main.payby = '$_'", @payby). ' )';
+  }
+
+  ##
+  # amounts
+  ##
+
+  #my $balance_sql = $class->balance_sql();
+  my $balance_sql = FS::cust_main->balance_sql();
+
+  push @where, map { s/current_balance/$balance_sql/; $_ }
+                   @{ $params->{'current_balance'} };
+
+  ##
+  # custbatch
+  ##
+
+  if ( $params->{'custbatch'} =~ /^([\w\/\-\:\.]+)$/ and $1 ) {
+    push @where,
+      "cust_main.custbatch = '$1'";
+  }
+
+  ##
+  # setup queries, subs, etc. for the search
+  ##
+
+  $orderby ||= 'ORDER BY custnum';
+
+  # here is the agent virtualization
+  push @where, $FS::CurrentUser::CurrentUser->agentnums_sql;
+
+  my $extra_sql = scalar(@where) ? ' WHERE '. join(' AND ', @where) : '';
+
+  my $addl_from = 'LEFT JOIN cust_pkg USING ( custnum  ) ';
+
+  my $count_query = "SELECT COUNT(*) FROM cust_main $extra_sql";
+
+  my $select = join(', ', 
+                 'cust_main.custnum',
+                 FS::UI::Web::cust_sql_fields($params->{'cust_fields'}),
+               );
+
+  my(@extra_headers) = ();
+  my(@extra_fields)  = ();
+
+  if ($params->{'flattened_pkgs'}) {
+
+    if ($dbh->{Driver}->{Name} eq 'Pg') {
+
+      $select .= ", array_to_string(array(select pkg from cust_pkg left join part_pkg using ( pkgpart ) where cust_main.custnum = cust_pkg.custnum $pkgwhere),'|') as magic";
+
+    }elsif ($dbh->{Driver}->{Name} =~ /^mysql/i) {
+      $select .= ", GROUP_CONCAT(pkg SEPARATOR '|') as magic";
+      $addl_from .= " LEFT JOIN part_pkg using ( pkgpart )";
+    }else{
+      warn "warning: unknown database type ". $dbh->{Driver}->{Name}. 
+           "omitting packing information from report.";
+    }
+
+    my $header_query = "SELECT COUNT(cust_pkg.custnum = cust_main.custnum) AS count FROM cust_main $addl_from $extra_sql $pkgwhere group by cust_main.custnum order by count desc limit 1";
+
+    my $sth = dbh->prepare($header_query) or die dbh->errstr;
+    $sth->execute() or die $sth->errstr;
+    my $headerrow = $sth->fetchrow_arrayref;
+    my $headercount = $headerrow ? $headerrow->[0] : 0;
+    while($headercount) {
+      unshift @extra_headers, "Package ". $headercount;
+      unshift @extra_fields, eval q!sub {my $c = shift;
+                                         my @a = split '\|', $c->magic;
+                                         my $p = $a[!.--$headercount. q!];
+                                         $p;
+                                        };!;
+    }
+
+  }
+
+  my $sql_query = {
+    'table'         => 'cust_main',
+    'select'        => $select,
+    'hashref'       => {},
+    'extra_sql'     => $extra_sql,
+    'order_by'      => $orderby,
+    'count_query'   => $count_query,
+    'extra_headers' => \@extra_headers,
+    'extra_fields'  => \@extra_fields,
+  };
+
+}
+
+=item email_search_sql HASHREF
+
+(Class method)
+
+Emails a notice to the specified customers.
+
+Valid parameters are those of the L<search_sql> method, plus the following:
+
+=over 4
+
+=item from
+
+From: address
+
+=item subject
+
+Email Subject:
+
+=item html_body
+
+HTML body
+
+=item text_body
+
+Text body
+
+=item job
+
+Optional job queue job for status updates.
+
+=back
+
+Returns an error message, or false for success.
+
+If an error occurs during any email, stops the enture send and returns that
+error.  Presumably if you're getting SMTP errors aborting is better than 
+retrying everything.
+
+=cut
+
+sub email_search_sql {
+  my($class, $params) = @_;
+
+  my $from = delete $params->{from};
+  my $subject = delete $params->{subject};
+  my $html_body = delete $params->{html_body};
+  my $text_body = delete $params->{text_body};
+
+  my $job = delete $params->{'job'};
+
+  my $sql_query = $class->search_sql($params);
+
+  my $count_query   = delete($sql_query->{'count_query'});
+  my $count_sth = dbh->prepare($count_query)
+    or die "Error preparing $count_query: ". dbh->errstr;
+  $count_sth->execute
+    or die "Error executing $count_query: ". $count_sth->errstr;
+  my $count_arrayref = $count_sth->fetchrow_arrayref;
+  my $num_cust = $count_arrayref->[0];
+
+  #my @extra_headers = @{ delete($sql_query->{'extra_headers'}) };
+  #my @extra_fields  = @{ delete($sql_query->{'extra_fields'})  };
+
+
+  my( $num, $last, $min_sec ) = (0, time, 5); #progresbar foo
+
+  #eventually order+limit magic to reduce memory use?
+  foreach my $cust_main ( qsearch($sql_query) ) {
+
+    my $to = $cust_main->invoicing_list_emailonly_scalar;
+    next unless $to;
+
+    my $error = send_email(
+      generate_email(
+        'from'      => $from,
+        'to'        => $to,
+        'subject'   => $subject,
+        'html_body' => $html_body,
+        'text_body' => $text_body,
+      )
+    );
+    return $error if $error;
+
+    if ( $job ) { #progressbar foo
+      $num++;
+      if ( time - $min_sec > $last ) {
+        my $error = $job->update_statustext(
+          int( 100 * $num / $num_cust )
+        );
+        die $error if $error;
+        $last = time;
+      }
+    }
+
+  }
+
+  return '';
+}
+
+use Storable qw(thaw);
+use Data::Dumper;
+use MIME::Base64;
+sub process_email_search_sql {
+  my $job = shift;
+  #warn "$me process_re_X $method for job $job\n" if $DEBUG;
+
+  my $param = thaw(decode_base64(shift));
+  warn Dumper($param) if $DEBUG;
+
+  $param->{'job'} = $job;
+
+  my $error = FS::cust_main->email_search_sql( $param );
+  die $error if $error;
+
+}
+
 =item fuzzy_search FUZZY_HASHREF [ HASHREF, SELECT, EXTRA_SQL, CACHE_OBJ ]
 
 Performs a fuzzy (approximate) search and returns the matching FS::cust_main
@@ -5328,10 +5983,14 @@ sub smart_search {
 
     }
 
-  } elsif ( $search =~ /^\s*(\d+)\s*$/ ) { # customer # search
-                                           # (also try agent_custid)
-                                           # (regex needs tweaking if your
-                                           #  legacy cust numbers have letters)
+  # custnum search (also try agent_custid), with some tweaking options if your
+  # legacy cust "numbers" have letters
+  } elsif ( $search =~ /^\s*(\d+)\s*$/
+            || ( $conf->config('cust_main-agent_custid-format') eq 'ww?d+'
+                 && $search =~ /^\s*(\w\w?\d+)\s*$/
+               )
+          )
+  {
 
     push @cust_main, qsearch( {
       'table'     => 'cust_main',
@@ -5437,7 +6096,7 @@ sub smart_search {
     #getting complaints searches are not returning enough
     unless ( @cust_main  && $skip_fuzzy || $conf->exists('disable-fuzzy') ) {
 
-      #still some false laziness w/ search/cust_main.cgi
+      #still some false laziness w/search_sql (was search/cust_main.cgi)
 
       #substring
 
@@ -5508,6 +6167,72 @@ sub smart_search {
 
 }
 
+=item email_search
+
+Accepts the following options: I<email>, the email address to search for.  The
+email address will be searched for as an email invoice destination and as an
+svc_acct account.
+
+#Any additional options are treated as an additional qualifier on the search
+#(i.e. I<agentnum>).
+
+Returns a (possibly empty) array of FS::cust_main objects (but usually just
+none or one).
+
+=cut
+
+sub email_search {
+  my %options = @_;
+
+  local($DEBUG) = 1;
+
+  my $email = delete $options{'email'};
+
+  #we're only being used by RT at the moment... no agent virtualization yet
+  #my $agentnums_sql = $FS::CurrentUser::CurrentUser->agentnums_sql;
+
+  my @cust_main = ();
+
+  if ( $email =~ /([^@]+)\@([^@]+)/ ) {
+
+    my ( $user, $domain ) = ( $1, $2 );
+
+    warn "$me smart_search: searching for $user in domain $domain"
+      if $DEBUG;
+
+    push @cust_main,
+      map $_->cust_main,
+          qsearch( {
+                     'table'     => 'cust_main_invoice',
+                     'hashref'   => { 'dest' => $email },
+                   }
+                 );
+
+    push @cust_main,
+      map  $_->cust_main,
+      grep $_,
+      map  $_->cust_svc->cust_pkg,
+          qsearch( {
+                     'table'     => 'svc_acct',
+                     'hashref'   => { 'username' => $user, },
+                     'extra_sql' =>
+                       'AND ( SELECT domain FROM svc_domain
+                                WHERE svc_acct.domsvc = svc_domain.svcnum
+                            ) = '. dbh->quote($domain),
+                   }
+                 );
+  }
+
+  my %saw = ();
+  @cust_main = grep { !$saw{$_->custnum}++ } @cust_main;
+
+  warn "$me smart_search: found ". scalar(@cust_main). " unique customers"
+    if $DEBUG;
+
+  @cust_main;
+
+}
+
 =item check_and_rebuild_fuzzyfiles
 
 =cut
@@ -5609,21 +6334,79 @@ sub append_fuzzyfiles {
   1;
 }
 
+=item process_batch_import
+
+Load a batch import as a queued JSRPC job
+
+=cut
+
+use Storable qw(thaw);
+use Data::Dumper;
+use MIME::Base64;
+sub process_batch_import {
+  my $job = shift;
+
+  my $param = thaw(decode_base64(shift));
+  warn Dumper($param) if $DEBUG;
+  
+  my $files = $param->{'uploaded_files'}
+    or die "No files provided.\n";
+
+  my (%files) = map { /^(\w+):([\.\w]+)$/ ? ($1,$2):() } split /,/, $files;
+
+  my $dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc. '/';
+  my $file = $dir. $files{'file'};
+
+  my $type;
+  if ( $file =~ /\.(\w+)$/i ) {
+    $type = lc($1);
+  } else {
+    #or error out???
+    warn "can't parse file type from filename $file; defaulting to CSV";
+    $type = 'csv';
+  }
+
+  my $error =
+    FS::cust_main::batch_import( {
+      job       => $job,
+      file      => $file,
+      type      => $type,
+      custbatch => $param->{custbatch},
+      agentnum  => $param->{'agentnum'},
+      refnum    => $param->{'refnum'},
+      pkgpart   => $param->{'pkgpart'},
+      #'fields'  => [qw( cust_pkg.setup dayphone first last address1 address2
+      #                 city state zip comments                          )],
+      'format'  => $param->{'format'},
+    } );
+
+  unlink $file;
+
+  die "$error\n" if $error;
+
+}
+
 =item batch_import
 
 =cut
 
+#some false laziness w/cdr.pm now
 sub batch_import {
   my $param = shift;
-  #warn join('-',keys %$param);
-  my $fh = $param->{filehandle};
-  my $agentnum = $param->{agentnum};
 
-  my $refnum = $param->{refnum};
-  my $pkgpart = $param->{pkgpart};
+  my $job       = $param->{job};
+
+  my $filename  = $param->{file};
+  my $type      = $param->{type} || 'csv';
+
+  my $custbatch = $param->{custbatch};
+
+  my $agentnum  = $param->{agentnum};
+  my $refnum    = $param->{refnum};
+  my $pkgpart   = $param->{pkgpart};
+
+  my $format    = $param->{'format'};
 
-  #my @fields = @{$param->{fields}};
-  my $format = $param->{'format'};
   my @fields;
   my $payby;
   if ( $format eq 'simple' ) {
@@ -5658,14 +6441,34 @@ sub batch_import {
     die "unknown format $format";
   }
 
-  eval "use Text::CSV_XS;";
-  die $@ if $@;
+  my $count;
+  my $parser;
+  my @buffer = ();
+  if ( $type eq 'csv' ) {
 
-  my $csv = new Text::CSV_XS;
-  #warn $csv;
-  #warn $fh;
+    eval "use Text::CSV_XS;";
+    die $@ if $@;
+
+    $parser = new Text::CSV_XS;
+
+    @buffer = split(/\r?\n/, slurp($filename) );
+    $count = scalar(@buffer);
+
+  } elsif ( $type eq 'xls' ) {
+
+    eval "use Spreadsheet::ParseExcel;";
+    die $@ if $@;
+
+    my $excel = new Spreadsheet::ParseExcel::Workbook->Parse($filename);
+    $parser = $excel->{Worksheet}[0]; #first sheet
+
+    $count = $parser->{MaxRow} || $parser->{MinRow};
+    $count++;
+
+  } else {
+    die "Unknown file type $type\n";
+  }
 
-  my $imported = 0;
   #my $columns;
 
   local $SIG{HUP} = 'IGNORE';
@@ -5679,24 +6482,46 @@ sub batch_import {
   local $FS::UID::AutoCommit = 0;
   my $dbh = dbh;
   
-  #while ( $columns = $csv->getline($fh) ) {
   my $line;
-  while ( defined($line=<$fh>) ) {
+  my $row = 0;
+  my( $last, $min_sec ) = ( time, 5 ); #progressbar foo
+  while (1) {
 
-    $csv->parse($line) or do {
-      $dbh->rollback if $oldAutoCommit;
-      return "can't parse: ". $csv->error_input();
-    };
+    my @columns = ();
+    if ( $type eq 'csv' ) {
+
+      last unless scalar(@buffer);
+      $line = shift(@buffer);
+
+      $parser->parse($line) or do {
+        $dbh->rollback if $oldAutoCommit;
+        return "can't parse: ". $parser->error_input();
+      };
+      @columns = $parser->fields();
+
+    } elsif ( $type eq 'xls' ) {
+
+      last if $row > ($parser->{MaxRow} || $parser->{MinRow});
+
+      my @row = @{ $parser->{Cells}[$row] };
+      @columns = map $_->{Val}, @row;
+
+      #my $z = 'A';
+      #warn $z++. ": $_\n" for @columns;
+
+    } else {
+      die "Unknown file type $type\n";
+    }
 
-    my @columns = $csv->fields();
     #warn join('-',@columns);
 
     my %cust_main = (
-      agentnum => $agentnum,
-      refnum   => $refnum,
-      country  => $conf->config('countrydefault') || 'US',
-      payby    => $payby, #default
-      paydate  => '12/2037', #default
+      custbatch => $custbatch,
+      agentnum  => $agentnum,
+      refnum    => $refnum,
+      country   => $conf->config('countrydefault') || 'US',
+      payby     => $payby, #default
+      paydate   => '12/2037', #default
     );
     my $billtime = time;
     my %cust_pkg = ( pkgpart => $pkgpart );
@@ -5743,12 +6568,14 @@ sub batch_import {
           $columns[0] = $part_referral->refnum;
         }
 
-        #$cust_main{$field} = shift @$columns; 
-        $cust_main{$field} = shift @columns; 
+        my $value = shift @columns;
+        $cust_main{$field} = $value if length($value);
       }
     }
 
-    $cust_main{'payby'} = 'CARD' if length($cust_main{'payinfo'});
+    $cust_main{'payby'} = 'CARD'
+      if defined $cust_main{'payinfo'}
+      && length  $cust_main{'payinfo'};
 
     my $invoicing_list = $cust_main{'invoicing_list'}
                            ? [ delete $cust_main{'invoicing_list'} ]
@@ -5780,7 +6607,7 @@ sub batch_import {
 
     if ( $error ) {
       $dbh->rollback if $oldAutoCommit;
-      return "can't insert customer for $line: $error";
+      return "can't insert customer". ( $line ? " for $line" : '' ). ": $error";
     }
 
     if ( $format eq 'simple' ) {
@@ -5806,12 +6633,18 @@ sub batch_import {
 
     }
 
-    $imported++;
+    $row++;
+
+    if ( $job && time - $min_sec > $last ) { #progress bar
+      $job->update_statustext( int(100 * $row / $count) );
+      $last = time;
+    }
+
   }
 
-  $dbh->commit or die $dbh->errstr if $oldAutoCommit;
+  $dbh->commit or die $dbh->errstr if $oldAutoCommit;;
 
-  return "Empty file!" unless $imported;
+  return "Empty file!" unless $row;
 
   ''; #no error