X-Git-Url: http://git.freeside.biz/gitweb/?a=blobdiff_plain;f=FS%2FFS%2Fcust_main.pm;h=718fccf2f1de56373e7104cf78402c0f4ea3b5da;hb=0269c850cfefc00d5da255f88c63a314e1ab6cd0;hp=b2b0a9b1c15f6677beeedbf607d28f45e6f9aee3;hpb=cd907554f4c08175c379e204e28e26483acf957a;p=freeside.git

diff --git a/FS/FS/cust_main.pm b/FS/FS/cust_main.pm
index b2b0a9b1c..718fccf2f 100644
--- a/FS/FS/cust_main.pm
+++ b/FS/FS/cust_main.pm
@@ -1,5 +1,6 @@
 package FS::cust_main;
 
+require 5.006;
 use strict;
 use vars qw( @ISA @EXPORT_OK $DEBUG $me $conf @encrypted_fields
              $import $skip_fuzzyfiles $ignore_expired_card @paytypes);
@@ -7,35 +8,36 @@ use vars qw( $realtime_bop_decline_quiet ); #ugh
 use Safe;
 use Carp;
 use Exporter;
-BEGIN {
-  eval "use Time::Local;";
-  die "Time::Local minimum version 1.05 required with Perl versions before 5.6"
-    if $] < 5.006 && !defined($Time::Local::VERSION);
-  #eval "use Time::Local qw(timelocal timelocal_nocheck);";
-  eval "use Time::Local qw(timelocal_nocheck);";
-}
+use Scalar::Util qw( blessed );
+use Time::Local qw(timelocal_nocheck);
+use Data::Dumper;
+use Tie::IxHash;
 use Digest::MD5 qw(md5_base64);
 use Date::Format;
 use Date::Parse;
 #use Date::Manip;
+use File::Slurp qw( slurp );
+use File::Temp qw( tempfile );
 use String::Approx qw(amatch);
 use Business::CreditCard 0.28;
 use Locale::Country;
-use Data::Dumper;
-use FS::UID qw( getotaker dbh );
+use FS::UID qw( getotaker dbh driver_name );
 use FS::Record qw( qsearchs qsearch dbdef );
-use FS::Misc qw( send_email generate_ps do_print );
+use FS::Misc qw( generate_email send_email generate_ps do_print );
 use FS::Msgcat qw(gettext);
 use FS::cust_pkg;
 use FS::cust_svc;
 use FS::cust_bill;
 use FS::cust_bill_pkg;
 use FS::cust_pay;
+use FS::cust_pay_pending;
 use FS::cust_pay_void;
+use FS::cust_pay_batch;
 use FS::cust_credit;
 use FS::cust_refund;
 use FS::part_referral;
 use FS::cust_main_county;
+use FS::cust_tax_location;
 use FS::agent;
 use FS::cust_main_invoice;
 use FS::cust_credit_bill;
@@ -43,17 +45,17 @@ use FS::cust_bill_pay;
 use FS::prepay_credit;
 use FS::queue;
 use FS::part_pkg;
-use FS::part_bill_event qw(due_events);
-use FS::cust_bill_event;
-use FS::cust_tax_exempt;
-use FS::cust_tax_exempt_pkg;
+use FS::part_event;
+use FS::part_event_condition;
+#use FS::cust_event;
 use FS::type_pkgs;
 use FS::payment_gateway;
 use FS::agent_payment_gateway;
 use FS::banned_pay;
 use FS::payinfo_Mixin;
+use FS::TicketSystem;
 
-@ISA = qw( FS::Record FS::payinfo_Mixin );
+@ISA = qw( FS::payinfo_Mixin FS::Record );
 
 @EXPORT_OK = qw( smart_search );
 
@@ -224,6 +226,8 @@ Card Verification Value, "CVV2" (also known as CVC2 or CID), the 3 or 4 digit nu
 
 =item spool_cdr - Enable individual CDR spooling, empty or `Y'
 
+=item squelch_cdr - Discourage individual CDR printing, empty or `Y'
+
 =back
 
 =head1 METHODS
@@ -350,7 +354,8 @@ sub insert {
     $error = $self->check_invoicing_list( $invoicing_list );
     if ( $error ) {
       $dbh->rollback if $oldAutoCommit;
-      return "checking invoicing_list (transaction rolled back): $error";
+      #return "checking invoicing_list (transaction rolled back): $error";
+      return $error;
     }
     $self->invoicing_list( $invoicing_list );
   }
@@ -1052,7 +1057,7 @@ sub delete {
 
 }
 
-=item replace OLD_RECORD [ INVOICING_LIST_ARYREF ]
+=item replace [ OLD_RECORD ] [ INVOICING_LIST_ARYREF ]
 
 Replaces the OLD_RECORD with this one in the database.  If there is an error,
 returns the error, otherwise returns false.
@@ -1068,23 +1073,16 @@ check_invoicing_list first.  Here's an example:
 
 sub replace {
   my $self = shift;
-  my $old = shift;
+
+  my $old = ( blessed($_[0]) && $_[0]->isa('FS::Record') )
+              ? shift
+              : $self->replace_old;
+
   my @param = @_;
+
   warn "$me replace called\n"
     if $DEBUG;
 
-  local $SIG{HUP} = 'IGNORE';
-  local $SIG{INT} = 'IGNORE';
-  local $SIG{QUIT} = 'IGNORE';
-  local $SIG{TERM} = 'IGNORE';
-  local $SIG{TSTP} = 'IGNORE';
-  local $SIG{PIPE} = 'IGNORE';
-
-  # We absolutely have to have an old vs. new record to make this work.
-  if (!defined($old)) {
-    $old = qsearchs( 'cust_main', { 'custnum' => $self->custnum } );
-  }
-
   my $curuser = $FS::CurrentUser::CurrentUser;
   if (    $self->payby eq 'COMP'
        && $self->payby ne $old->payby
@@ -1099,6 +1097,13 @@ sub replace {
     && $self->payby =~ /^(CARD|DCRD)$/
     && ( $old->payinfo eq $self->payinfo || $old->paymask eq $self->paymask );
 
+  local $SIG{HUP} = 'IGNORE';
+  local $SIG{INT} = 'IGNORE';
+  local $SIG{QUIT} = 'IGNORE';
+  local $SIG{TERM} = 'IGNORE';
+  local $SIG{TSTP} = 'IGNORE';
+  local $SIG{PIPE} = 'IGNORE';
+
   my $oldAutoCommit = $FS::UID::AutoCommit;
   local $FS::UID::AutoCommit = 0;
   my $dbh = dbh;
@@ -1207,6 +1212,7 @@ sub check {
     || $self->ut_number('agentnum')
     || $self->ut_textn('agent_custid')
     || $self->ut_number('refnum')
+    || $self->ut_textn('custbatch')
     || $self->ut_name('last')
     || $self->ut_name('first')
     || $self->ut_snumbern('birthdate')
@@ -1274,58 +1280,75 @@ sub check {
   ;
   return $error if $error;
 
-  my @addfields = qw(
-    last first company address1 address2 city county state zip
-    country daytime night fax
-  );
+  if ( $conf->exists('cust_main-require_phone')
+       && ! length($self->daytime) && ! length($self->night)
+     ) {
 
-  if ( defined $self->dbdef_table->column('ship_last') ) {
-    if ( scalar ( grep { $self->getfield($_) ne $self->getfield("ship_$_") }
-                       @addfields )
-         && scalar ( grep { $self->getfield("ship_$_") ne '' } @addfields )
-       )
-    {
-      my $error =
-        $self->ut_name('ship_last')
-        || $self->ut_name('ship_first')
-        || $self->ut_textn('ship_company')
-        || $self->ut_text('ship_address1')
-        || $self->ut_textn('ship_address2')
-        || $self->ut_text('ship_city')
-        || $self->ut_textn('ship_county')
-        || $self->ut_textn('ship_state')
-        || $self->ut_country('ship_country')
-      ;
-      return $error if $error;
+    my $daytime_label = FS::Msgcat::_gettext('daytime') =~ /^(daytime)?$/
+                          ? 'Day Phone'
+                          : FS::Msgcat::_gettext('daytime');
+    my $night_label = FS::Msgcat::_gettext('night') =~ /^(night)?$/
+                        ? 'Night Phone'
+                        : FS::Msgcat::_gettext('night');
+  
+    return "$daytime_label or $night_label is required"
+  
+  }
 
-      #false laziness with above
-      unless ( qsearchs('cust_main_county', {
-        'country' => $self->ship_country,
-        'state'   => '',
-       } ) ) {
-        return "Unknown ship_state/ship_county/ship_country: ".
-          $self->ship_state. "/". $self->ship_county. "/". $self->ship_country
-          unless qsearch('cust_main_county',{
-            'state'   => $self->ship_state,
-            'county'  => $self->ship_county,
-            'country' => $self->ship_country,
-          } );
-      }
-      #eofalse
-
-      $error =
-        $self->ut_phonen('ship_daytime', $self->ship_country)
-        || $self->ut_phonen('ship_night', $self->ship_country)
-        || $self->ut_phonen('ship_fax', $self->ship_country)
-        || $self->ut_zip('ship_zip', $self->ship_country)
-      ;
-      return $error if $error;
+  if ( $self->has_ship_address
+       && scalar ( grep { $self->getfield($_) ne $self->getfield("ship_$_") }
+                        $self->addr_fields )
+     )
+  {
+    my $error =
+      $self->ut_name('ship_last')
+      || $self->ut_name('ship_first')
+      || $self->ut_textn('ship_company')
+      || $self->ut_text('ship_address1')
+      || $self->ut_textn('ship_address2')
+      || $self->ut_text('ship_city')
+      || $self->ut_textn('ship_county')
+      || $self->ut_textn('ship_state')
+      || $self->ut_country('ship_country')
+    ;
+    return $error if $error;
 
-    } else { # ship_ info eq billing info, so don't store dup info in database
-      $self->setfield("ship_$_", '')
-        foreach qw( last first company address1 address2 city county state zip
-                    country daytime night fax );
+    #false laziness with above
+    unless ( qsearchs('cust_main_county', {
+      'country' => $self->ship_country,
+      'state'   => '',
+     } ) ) {
+      return "Unknown ship_state/ship_county/ship_country: ".
+        $self->ship_state. "/". $self->ship_county. "/". $self->ship_country
+        unless qsearch('cust_main_county',{
+          'state'   => $self->ship_state,
+          'county'  => $self->ship_county,
+          'country' => $self->ship_country,
+        } );
     }
+    #eofalse
+
+    $error =
+      $self->ut_phonen('ship_daytime', $self->ship_country)
+      || $self->ut_phonen('ship_night', $self->ship_country)
+      || $self->ut_phonen('ship_fax', $self->ship_country)
+      || $self->ut_zip('ship_zip', $self->ship_country)
+    ;
+    return $error if $error;
+
+    return "Unit # is required."
+      if $self->ship_address2 =~ /^\s*$/
+      && $conf->exists('cust_main-require_address2');
+
+  } else { # ship_ info eq billing info, so don't store dup info in database
+
+    $self->setfield("ship_$_", '')
+      foreach $self->addr_fields;
+
+    return "Unit # is required."
+      if $self->address2 =~ /^\s*$/
+      && $conf->exists('cust_main-require_address2');
+
   }
 
   #$self->payby =~ /^(CARD|DCRD|CHEK|DCHK|LECB|BILL|COMP|PREPAY|CASH|WEST|MCRD)$/
@@ -1513,7 +1536,7 @@ sub check {
     $self->payname($1);
   }
 
-  foreach my $flag (qw( tax spool_cdr )) {
+  foreach my $flag (qw( tax spool_cdr squelch_cdr )) {
     $self->$flag() =~ /^(Y?)$/ or return "Illegal $flag: ". $self->$flag();
     $self->$flag($1);
   }
@@ -1526,6 +1549,30 @@ sub check {
   $self->SUPER::check;
 }
 
+=item addr_fields 
+
+Returns a list of fields which have ship_ duplicates.
+
+=cut
+
+sub addr_fields {
+  qw( last first company
+      address1 address2 city county state zip country
+      daytime night fax
+    );
+}
+
+=item has_ship_address
+
+Returns true if this customer record has a separate shipping address.
+
+=cut
+
+sub has_ship_address {
+  my $self = shift;
+  scalar( grep { $self->getfield("ship_$_") ne '' } $self->addr_fields );
+}
+
 =item all_pkgs
 
 Returns all packages (see L<FS::cust_pkg>) for this customer.
@@ -1547,6 +1594,16 @@ sub all_pkgs {
   sort sort_packages @cust_pkg;
 }
 
+=item cust_pkg
+
+Synonym for B<all_pkgs>.
+
+=cut
+
+sub cust_pkg {
+  shift->all_pkgs(@_);
+}
+
 =item ncancelled_pkgs
 
 Returns all non-cancelled packages (see L<FS::cust_pkg>) for this customer.
@@ -1561,11 +1618,18 @@ sub ncancelled_pkgs {
   my @cust_pkg = ();
   if ( $self->{'_pkgnum'} ) {
 
+    warn "$me ncancelled_pkgs: returning cached objects"
+      if $DEBUG > 1;
+
     @cust_pkg = grep { ! $_->getfield('cancel') }
                 values %{ $self->{'_pkgnum'}->cache };
 
   } else {
 
+    warn "$me ncancelled_pkgs: searching for packages with custnum ".
+         $self->custnum. "\n"
+      if $DEBUG > 1;
+
     @cust_pkg =
       qsearch( 'cust_pkg', {
                              'custnum' => $self->custnum,
@@ -1648,7 +1712,8 @@ sub num_ncancelled_pkgs {
 }
 
 sub num_pkgs {
-  my( $self, $sql ) = @_;
+  my( $self ) = shift;
+  my $sql = scalar(@_) ? shift : '';
   $sql = "AND $sql" if $sql && $sql !~ /^\s*$/ && $sql !~ /^\s*AND/i;
   my $sth = dbh->prepare(
     "SELECT COUNT(*) FROM cust_pkg WHERE custnum = ? $sql"
@@ -1683,10 +1748,20 @@ sub suspend {
   grep { $_->suspend(@_) } $self->unsuspended_pkgs;
 }
 
-=item suspend_if_pkgpart PKGPART [ , PKGPART ... ]
+=item suspend_if_pkgpart HASHREF | PKGPART [ , PKGPART ... ]
 
 Suspends all unsuspended packages (see L<FS::cust_pkg>) matching the listed
-PKGPARTs (see L<FS::part_pkg>).
+PKGPARTs (see L<FS::part_pkg>).  Preferred usage is to pass a hashref instead
+of a list of pkgparts; the hashref has the following keys:
+
+=over 4
+
+=item pkgparts - listref of pkgparts
+
+=item (other options are passed to the suspend method)
+
+=back
+
 
 Returns a list: an empty list on success or a list of errors.
 
@@ -1706,10 +1781,19 @@ sub suspend_if_pkgpart {
       $self->unsuspended_pkgs;
 }
 
-=item suspend_unless_pkgpart PKGPART [ , PKGPART ... ]
+=item suspend_unless_pkgpart HASHREF | PKGPART [ , PKGPART ... ]
 
 Suspends all unsuspended packages (see L<FS::cust_pkg>) unless they match the
-listed PKGPARTs (see L<FS::part_pkg>).
+given PKGPARTs (see L<FS::part_pkg>).  Preferred usage is to pass a hashref
+instead of a list of pkgparts; the hashref has the following keys:
+
+=over 4
+
+=item pkgparts - listref of pkgparts
+
+=item (other options are passed to the suspend method)
+
+=back
 
 Returns a list: an empty list on success or a list of errors.
 
@@ -1733,22 +1817,31 @@ sub suspend_unless_pkgpart {
 
 Cancels all uncancelled packages (see L<FS::cust_pkg>) for this customer.
 
-Available options are: I<quiet>, I<reasonnum>, and I<ban>
+Available options are:
+
+=over 4
+
+=item quiet - can be set true to supress email cancellation notices.
 
-I<quiet> can be set true to supress email cancellation notices.
+=item reason - can be set to a cancellation reason (see L<FS:reason>), either a reasonnum of an existing reason, or passing a hashref will create a new reason.  The hashref should have the following keys: typenum - Reason type (see L<FS::reason_type>, reason - Text of the new reason.
 
-# I<reasonnum> can be set to a cancellation reason (see L<FS::cancel_reason>)
+=item ban - can be set true to ban this customer's credit card or ACH information, if present.
 
-I<ban> can be set true to ban this customer's credit card or ACH information,
-if present.
+=back
 
 Always returns a list: an empty list on success or a list of errors.
 
 =cut
 
 sub cancel {
-  my $self = shift;
-  my %opt = @_;
+  my( $self, %opt ) = @_;
+
+  warn "$me cancel called on customer ". $self->custnum. " with options ".
+       join(', ', map { "$_: $opt{$_}" } keys %opt ). "\n"
+    if $DEBUG;
+
+  return ( 'access denied' )
+    unless $FS::CurrentUser::CurrentUser->access_right('Cancel customer');
 
   if ( $opt{'ban'} && $self->payby =~ /^(CARD|DCRD|CHEK|DCHK)$/ ) {
 
@@ -1763,7 +1856,13 @@ sub cancel {
 
   }
 
-  grep { $_ } map { $_->cancel(@_) } $self->ncancelled_pkgs;
+  my @pkgs = $self->ncancelled_pkgs;
+
+  warn "$me cancelling ". scalar($self->ncancelled_pkgs). "/".
+       scalar(@pkgs). " packages for customer ". $self->custnum. "\n"
+    if $DEBUG;
+
+  grep { $_ } map { $_->cancel(%opt) } $self->ncancelled_pkgs;
 }
 
 sub _banned_pay_hashref {
@@ -1810,10 +1909,108 @@ sub agent {
   qsearchs( 'agent', { 'agentnum' => $self->agentnum } );
 }
 
+=item bill_and_collect 
+
+Cancels and suspends any packages due, generates bills, applies payments and
+cred
+
+Warns on errors (Does not currently: If there is an error, returns the error, otherwise returns false.)
+
+Options are passed as name-value pairs.  Currently available options are:
+
+=over 4
+
+=item time
+
+Bills the customer as if it were that time.  Specified as a UNIX timestamp; see L<perlfunc/"time">).  Also see L<Time::Local> and L<Date::Parse> for conversion functions.  For example:
+
+ use Date::Parse;
+ ...
+ $cust_main->bill( 'time' => str2time('April 20th, 2001') );
+
+=item invoice_time
+
+Used in conjunction with the I<time> option, this option specifies the date of for the generated invoices.  Other calculations, such as whether or not to generate the invoice in the first place, are not affected.
+
+=item check_freq
+
+"1d" for the traditional, daily events (the default), or "1m" for the new monthly events (part_event.check_freq)
+
+=item resetup
+
+If set true, re-charges setup fees.
+
+=item debug
+
+Debugging level.  Default is 0 (no debugging), or can be set to 1 (passed-in options), 2 (traces progress), 3 (more information), or 4 (include full search queries)
+
+=back
+
+=cut
+
+sub bill_and_collect {
+  my( $self, %options ) = @_;
+
+  ###
+  # cancel packages
+  ###
+
+  #$options{actual_time} not $options{time} because freeside-daily -d is for
+  #pre-printing invoices
+  my @cancel_pkgs = grep { $_->expire && $_->expire <= $options{actual_time} }
+                         $self->ncancelled_pkgs;
+
+  foreach my $cust_pkg ( @cancel_pkgs ) {
+    my $error = $cust_pkg->cancel;
+    warn "Error cancelling expired pkg ". $cust_pkg->pkgnum.
+         " for custnum ". $self->custnum. ": $error"
+      if $error;
+  }
+
+  ###
+  # suspend packages
+  ###
+
+  #$options{actual_time} not $options{time} because freeside-daily -d is for
+  #pre-printing invoices
+  my @susp_pkgs = 
+    grep { ! $_->susp
+           && (    (    $_->part_pkg->is_prepaid
+                     && $_->bill
+                     && $_->bill < $options{actual_time}
+                   )
+                || (    $_->adjourn
+                    && $_->adjourn <= $options{actual_time}
+                  )
+              )
+         }
+         $self->ncancelled_pkgs;
+
+  foreach my $cust_pkg ( @susp_pkgs ) {
+    my $error = $cust_pkg->suspend;
+    warn "Error suspending package ". $cust_pkg->pkgnum.
+         " for custnum ". $self->custnum. ": $error"
+      if $error;
+  }
+
+  ###
+  # bill and collect
+  ###
+
+  my $error = $self->bill( %options );
+  warn "Error billing, custnum ". $self->custnum. ": $error" if $error;
+
+  $self->apply_payments_and_credits;
+
+  $error = $self->collect( %options );
+  warn "Error collecting, custnum". $self->custnum. ": $error" if $error;
+
+}
+
 =item bill OPTIONS
 
 Generates invoices (see L<FS::cust_bill>) for this customer.  Usually used in
-conjunction with the collect method.
+conjunction with the collect method by calling B<bill_and_collect>.
 
 If there is an error, returns the error, otherwise returns false.
 
@@ -1821,15 +2018,27 @@ Options are passed as name-value pairs.  Currently available options are:
 
 =over 4
 
-=item resetup - if set true, re-charges setup fees.
+=item resetup
+
+If set true, re-charges setup fees.
 
-=item time - bills the customer as if it were that time.  Specified as a UNIX timestamp; see L<perlfunc/"time">).  Also see L<Time::Local> and L<Date::Parse> for conversion functions.  For example:
+=item time
+
+Bills the customer as if it were that time.  Specified as a UNIX timestamp; see L<perlfunc/"time">).  Also see L<Time::Local> and L<Date::Parse> for conversion functions.  For example:
 
  use Date::Parse;
  ...
  $cust_main->bill( 'time' => str2time('April 20th, 2001') );
 
-=item invoice_time - used in conjunction with the I<time> option, this option specifies the date of for the generated invoices.  Other calculations, such as whether or not to generate the invoice in the first place, are not affected.
+=item pkg_list
+
+An array ref of specific packages (objects) to attempt billing, instead trying all of them.
+
+ $cust_main->bill( pkg_list => [$pkg1, $pkg2] );
+
+=item invoice_time
+
+Used in conjunction with the I<time> option, this option specifies the date of for the generated invoices.  Other calculations, such as whether or not to generate the invoice in the first place, are not affected.
 
 =back
 
@@ -1838,13 +2047,12 @@ Options are passed as name-value pairs.  Currently available options are:
 sub bill {
   my( $self, %options ) = @_;
   return '' if $self->payby eq 'COMP';
+  local $DEBUG = 1;
   warn "$me bill customer ". $self->custnum. "\n"
     if $DEBUG;
 
   my $time = $options{'time'} || time;
 
-  my $error;
-
   #put below somehow?
   local $SIG{HUP} = 'IGNORE';
   local $SIG{INT} = 'IGNORE';
@@ -1859,34 +2067,22 @@ sub bill {
 
   $self->select_for_update; #mutex
 
-  #create a new invoice
-  #(we'll remove it later if it doesn't actually need to be generated [contains
-  # no line items] and we're inside a transaciton so nothing else will see it)
-  my $cust_bill = new FS::cust_bill ( {
-    'custnum' => $self->custnum,
-    '_date'   => ( $options{'invoice_time'} || $time ),
-    #'charged' => $charged,
-    'charged' => 0,
-  } );
-  $error = $cust_bill->insert;
-  if ( $error ) {
-    $dbh->rollback if $oldAutoCommit;
-    return "can't create invoice for customer #". $self->custnum. ": $error";
-  }
-  my $invnum = $cust_bill->invnum;
+  my @cust_bill_pkg = ();
+  my @appended_cust_bill_pkg = ();
 
   ###
   # find the packages which are due for billing, find out how much they are
   # & generate invoice database.
   ###
 
-  my( $total_setup, $total_recur ) = ( 0, 0 );
+  my( $total_setup, $total_recur, $postal_charge ) = ( 0, 0, 0 );
   my %tax;
+  my %taxlisthash;
+  my %taxname;
   my @precommit_hooks = ();
 
-  foreach my $cust_pkg (
-    qsearch('cust_pkg', { 'custnum' => $self->custnum } )
-  ) {
+  my @cust_pkgs = qsearch('cust_pkg', { 'custnum' => $self->custnum } );
+  foreach my $cust_pkg (@cust_pkgs) {
 
     #NO!! next if $cust_pkg->cancel;  
     next if $cust_pkg->getfield('cancel');  
@@ -1897,338 +2093,191 @@ sub bill {
     $cust_pkg->setfield('bill', '')
       unless defined($cust_pkg->bill);
  
-    my $part_pkg = $cust_pkg->part_pkg;
+    #my $part_pkg = $cust_pkg->part_pkg;
 
+    my $real_pkgpart = $cust_pkg->pkgpart;
     my %hash = $cust_pkg->hash;
-    my $old_cust_pkg = new FS::cust_pkg \%hash;
-
-    my @details = ();
-
-    ###
-    # bill setup
-    ###
-
-    my $setup = 0;
-    if ( ! $cust_pkg->setup &&
-         (
-           ( $conf->exists('disable_setup_suspended_pkgs') &&
-            ! $cust_pkg->getfield('susp')
-          ) || ! $conf->exists('disable_setup_suspended_pkgs')
-         )
-      || $options{'resetup'}
-    ) {
-    
-      warn "    bill setup\n" if $DEBUG > 1;
-
-      $setup = eval { $cust_pkg->calc_setup( $time, \@details ) };
-      if ( $@ ) {
-        $dbh->rollback if $oldAutoCommit;
-        return "$@ running calc_setup for $cust_pkg\n";
-      }
-
-      $cust_pkg->setfield('setup', $time) unless $cust_pkg->setup;
-    }
-
-    ###
-    # bill recurring fee
-    ### 
-
-    my $recur = 0;
-    my $sdate;
-    if ( $part_pkg->getfield('freq') ne '0' &&
-         ! $cust_pkg->getfield('susp') &&
-         ( $cust_pkg->getfield('bill') || 0 ) <= $time
-    ) {
 
-      warn "    bill recur\n" if $DEBUG > 1;
+    foreach my $part_pkg ( $cust_pkg->part_pkg->self_and_bill_linked ) {
 
-      # XXX shared with $recur_prog
-      $sdate = $cust_pkg->bill || $cust_pkg->setup || $time;
-
-      #over two params!  lets at least switch to a hashref for the rest...
-      my %param = ( 'precommit_hooks' => \@precommit_hooks, );
-
-      $recur = eval { $cust_pkg->calc_recur( \$sdate, \@details, \%param ) };
-      if ( $@ ) {
-        $dbh->rollback if $oldAutoCommit;
-        return "$@ running calc_recur for $cust_pkg\n";
-      }
+      $cust_pkg->set($_, $hash{$_}) foreach qw ( setup last_bill bill );
 
-      #change this bit to use Date::Manip? CAREFUL with timezones (see
-      # mailing list archive)
-      my ($sec,$min,$hour,$mday,$mon,$year) =
-        (localtime($sdate) )[0,1,2,3,4,5];
-
-      #pro-rating magic - if $recur_prog fiddles $sdate, want to use that
-      # only for figuring next bill date, nothing else, so, reset $sdate again
-      # here
-      $sdate = $cust_pkg->bill || $cust_pkg->setup || $time;
-      $cust_pkg->last_bill($sdate)
-        if $cust_pkg->dbdef_table->column('last_bill');
-
-      if ( $part_pkg->freq =~ /^\d+$/ ) {
-        $mon += $part_pkg->freq;
-        until ( $mon < 12 ) { $mon -= 12; $year++; }
-      } elsif ( $part_pkg->freq =~ /^(\d+)w$/ ) {
-        my $weeks = $1;
-        $mday += $weeks * 7;
-      } elsif ( $part_pkg->freq =~ /^(\d+)d$/ ) {
-        my $days = $1;
-        $mday += $days;
-      } elsif ( $part_pkg->freq =~ /^(\d+)h$/ ) {
-        my $hours = $1;
-        $hour += $hours;
-      } else {
+      my $error =
+        $self->_make_lines( 'part_pkg'            => $part_pkg,
+                            'cust_pkg'            => $cust_pkg,
+                            'precommit_hooks'     => \@precommit_hooks,
+                            'line_items'          => \@cust_bill_pkg,
+                            'appended_line_items' => \@appended_cust_bill_pkg,
+                            'setup'               => \$total_setup,
+                            'recur'               => \$total_recur,
+                            'tax_matrix'          => \%taxlisthash,
+                            'time'                => $time,
+                            'options'             => \%options,
+                          );
+      if ($error) {
         $dbh->rollback if $oldAutoCommit;
-        return "unparsable frequency: ". $part_pkg->freq;
+        return $error;
       }
-      $cust_pkg->setfield('bill',
-        timelocal_nocheck($sec,$min,$hour,$mday,$mon,$year));
-    }
 
-    warn "\$setup is undefined" unless defined($setup);
-    warn "\$recur is undefined" unless defined($recur);
-    warn "\$cust_pkg->bill is undefined" unless defined($cust_pkg->bill);
+    } #foreach my $part_pkg
 
-    ###
-    # If $cust_pkg has been modified, update it and create cust_bill_pkg records
-    ###
-
-    if ( $cust_pkg->modified ) {  # hmmm.. and if the options are modified?
+  } #foreach my $cust_pkg
 
-      warn "  package ". $cust_pkg->pkgnum. " modified; updating\n"
-        if $DEBUG >1;
+  push @cust_bill_pkg, @appended_cust_bill_pkg;
 
-      $error=$cust_pkg->replace($old_cust_pkg,
-                                options => { $cust_pkg->options },
-                               );
-      if ( $error ) { #just in case
-        $dbh->rollback if $oldAutoCommit;
-        return "Error modifying pkgnum ". $cust_pkg->pkgnum. ": $error";
-      }
+  unless ( @cust_bill_pkg ) { #don't create an invoice w/o line items
+    #but do commit any package date cycling that happened
+    $dbh->commit or die $dbh->errstr if $oldAutoCommit;
+    return '';
+  }
 
-      $setup = sprintf( "%.2f", $setup );
-      $recur = sprintf( "%.2f", $recur );
-      if ( $setup < 0 && ! $conf->exists('allow_negative_charges') ) {
-        $dbh->rollback if $oldAutoCommit;
-        return "negative setup $setup for pkgnum ". $cust_pkg->pkgnum;
-      }
-      if ( $recur < 0 && ! $conf->exists('allow_negative_charges') ) {
+  my $postal_pkg = $self->charge_postal_fee();
+  if ( $postal_pkg && !ref( $postal_pkg ) ) {
+    $dbh->rollback if $oldAutoCommit;
+    return "can't charge postal invoice fee for customer ".
+      $self->custnum. ": $postal_pkg";
+  }
+  if ( $postal_pkg ) {
+    foreach my $part_pkg ( $postal_pkg->part_pkg->self_and_bill_linked ) {
+      my $error =
+        $self->_make_lines( 'part_pkg'            => $part_pkg,
+                            'cust_pkg'            => $postal_pkg,
+                            'precommit_hooks'     => \@precommit_hooks,
+                            'line_items'          => \@cust_bill_pkg,
+                            'appended_line_items' => \@appended_cust_bill_pkg,
+                            'setup'               => \$total_setup,
+                            'recur'               => \$total_recur,
+                            'tax_matrix'          => \%taxlisthash,
+                            'time'                => $time,
+                            'options'             => \%options,
+                          );
+      if ($error) {
         $dbh->rollback if $oldAutoCommit;
-        return "negative recur $recur for pkgnum ". $cust_pkg->pkgnum;
+        return $error;
       }
+    }
+  }
 
-      if ( $setup != 0 || $recur != 0 ) {
-
-        warn "    charges (setup=$setup, recur=$recur); adding line items\n"
-          if $DEBUG > 1;
-        my $cust_bill_pkg = new FS::cust_bill_pkg ({
-          'invnum'  => $invnum,
-          'pkgnum'  => $cust_pkg->pkgnum,
-          'setup'   => $setup,
-          'recur'   => $recur,
-          'sdate'   => $sdate,
-          'edate'   => $cust_pkg->bill,
-          'details' => \@details,
-        });
-        $error = $cust_bill_pkg->insert;
-        if ( $error ) {
-          $dbh->rollback if $oldAutoCommit;
-          return "can't create invoice line item for invoice #$invnum: $error";
-        }
-        $total_setup += $setup;
-        $total_recur += $recur;
-
-        ###
-        # handle taxes
-        ###
-
-        unless ( $self->tax =~ /Y/i || $self->payby eq 'COMP' ) {
-
-          my $prefix = 
-            ( $conf->exists('tax-ship_address') && length($self->ship_last) )
-            ? 'ship_'
-            : '';
-          my %taxhash = map { $_ => $self->get("$prefix$_") }
-                            qw( state county country );
-
-          $taxhash{'taxclass'} = $part_pkg->taxclass;
-
-          my @taxes = qsearch( 'cust_main_county', \%taxhash );
-
-          unless ( @taxes ) {
-            $taxhash{'taxclass'} = '';
-            @taxes =  qsearch( 'cust_main_county', \%taxhash );
-          }
-
-          #one more try at a whole-country tax rate
-          unless ( @taxes ) {
-            $taxhash{$_} = '' foreach qw( state county );
-            @taxes =  qsearch( 'cust_main_county', \%taxhash );
-          }
+  warn "having a look at the taxes we found...\n" if $DEBUG > 2;
+  foreach my $tax ( keys %taxlisthash ) {
+    my $tax_object = shift @{ $taxlisthash{$tax} };
+    warn "found ". $tax_object->taxname. " as $tax\n" if $DEBUG > 2;
+    my $listref_or_error = $tax_object->taxline( @{ $taxlisthash{$tax} } );
+    unless (ref($listref_or_error)) {
+      $dbh->rollback if $oldAutoCommit;
+      return $listref_or_error;
+    }
+    unshift @{ $taxlisthash{$tax} }, $tax_object;
 
-          # maybe eliminate this entirely, along with all the 0% records
-          unless ( @taxes ) {
-            $dbh->rollback if $oldAutoCommit;
-            return
-              "fatal: can't find tax rate for state/county/country/taxclass ".
-              join('/', ( map $self->get("$prefix$_"),
-                              qw(state county country)
-                        ),
-                        $part_pkg->taxclass ). "\n";
-          }
+    warn "adding ". $listref_or_error->[1].
+         " as ". $listref_or_error->[0]. "\n"
+      if $DEBUG > 2;
+    $tax{ $tax_object->taxname } += $listref_or_error->[1];
+    if ( $taxname{ $listref_or_error->[0] } ) {
+      push @{ $taxname{ $listref_or_error->[0] } }, $tax_object->taxname;
+    }else{
+      $taxname{ $listref_or_error->[0] } = [ $tax_object->taxname ];
+    }
   
-          foreach my $tax ( @taxes ) {
-
-            my $taxable_charged = 0;
-            $taxable_charged += $setup
-              unless $part_pkg->setuptax =~ /^Y$/i
-                  || $tax->setuptax =~ /^Y$/i;
-            $taxable_charged += $recur
-              unless $part_pkg->recurtax =~ /^Y$/i
-                  || $tax->recurtax =~ /^Y$/i;
-            next unless $taxable_charged;
-
-            if ( $tax->exempt_amount && $tax->exempt_amount > 0 ) {
-              #my ($mon,$year) = (localtime($sdate) )[4,5];
-              my ($mon,$year) = (localtime( $sdate || $cust_bill->_date ) )[4,5];
-              $mon++;
-              my $freq = $part_pkg->freq || 1;
-              if ( $freq !~ /(\d+)$/ ) {
-                $dbh->rollback if $oldAutoCommit;
-                return "daily/weekly package definitions not (yet?)".
-                       " compatible with monthly tax exemptions";
-              }
-              my $taxable_per_month =
-                sprintf("%.2f", $taxable_charged / $freq );
-
-              #call the whole thing off if this customer has any old
-              #exemption records...
-              my @cust_tax_exempt =
-                qsearch( 'cust_tax_exempt' => { custnum=> $self->custnum } );
-              if ( @cust_tax_exempt ) {
-                $dbh->rollback if $oldAutoCommit;
-                return
-                  'this customer still has old-style tax exemption records; '.
-                  'run bin/fs-migrate-cust_tax_exempt?';
-              }
+  }
 
-              foreach my $which_month ( 1 .. $freq ) {
-
-                #maintain the new exemption table now
-                my $sql = "
-                  SELECT SUM(amount)
-                    FROM cust_tax_exempt_pkg
-                      LEFT JOIN cust_bill_pkg USING ( billpkgnum )
-                      LEFT JOIN cust_bill     USING ( invnum     )
-                    WHERE custnum = ?
-                      AND taxnum  = ?
-                      AND year    = ?
-                      AND month   = ?
-                ";
-                my $sth = dbh->prepare($sql) or do {
-                  $dbh->rollback if $oldAutoCommit;
-                  return "fatal: can't lookup exising exemption: ". dbh->errstr;
-                };
-                $sth->execute(
-                  $self->custnum,
-                  $tax->taxnum,
-                  1900+$year,
-                  $mon,
-                ) or do {
-                  $dbh->rollback if $oldAutoCommit;
-                  return "fatal: can't lookup exising exemption: ". dbh->errstr;
-                };
-                my $existing_exemption = $sth->fetchrow_arrayref->[0] || 0;
-                
-                my $remaining_exemption =
-                  $tax->exempt_amount - $existing_exemption;
-                if ( $remaining_exemption > 0 ) {
-                  my $addl = $remaining_exemption > $taxable_per_month
-                    ? $taxable_per_month
-                    : $remaining_exemption;
-                  $taxable_charged -= $addl;
-
-                  my $cust_tax_exempt_pkg = new FS::cust_tax_exempt_pkg ( {
-                    'billpkgnum' => $cust_bill_pkg->billpkgnum,
-                    'taxnum'     => $tax->taxnum,
-                    'year'       => 1900+$year,
-                    'month'      => $mon,
-                    'amount'     => sprintf("%.2f", $addl ),
-                  } );
-                  $error = $cust_tax_exempt_pkg->insert;
-                  if ( $error ) {
-                    $dbh->rollback if $oldAutoCommit;
-                    return "fatal: can't insert cust_tax_exempt_pkg: $error";
-                  }
-                } # if $remaining_exemption > 0
-
-                #++
-                $mon++;
-                #until ( $mon < 12 ) { $mon -= 12; $year++; }
-                until ( $mon < 13 ) { $mon -= 12; $year++; }
-  
-              } #foreach $which_month
+  #some taxes are taxed
+  my %totlisthash;
   
-            } #if $tax->exempt_amount
-
-            $taxable_charged = sprintf( "%.2f", $taxable_charged);
-
-            #$tax += $taxable_charged * $cust_main_county->tax / 100
-            $tax{ $tax->taxname || 'Tax' } +=
-              $taxable_charged * $tax->tax / 100
-
-          } #foreach my $tax ( @taxes )
+  warn "finding taxed taxes...\n" if $DEBUG > 2;
+  foreach my $tax ( keys %taxlisthash ) {
+    my $tax_object = shift @{ $taxlisthash{$tax} };
+    warn "found possible taxed tax ". $tax_object->taxname. " we call $tax\n"
+      if $DEBUG > 2;
+    next unless $tax_object->can('tax_on_tax');
 
-        } #unless $self->tax =~ /Y/i || $self->payby eq 'COMP'
+    foreach my $tot ( $tax_object->tax_on_tax( $self ) ) {
+      my $totname = ref( $tot ). ' '. $tot->taxnum;
 
-      } #if $setup != 0 || $recur != 0
-      
-    } #if $cust_pkg->modified
+      warn "checking $totname which we call ". $tot->taxname. " as applicable\n"
+        if $DEBUG > 2;
+      next unless exists( $taxlisthash{ $totname } ); # only increase
+                                                      # existing taxes
+      warn "adding $totname to taxed taxes\n" if $DEBUG > 2;
+      if ( exists( $totlisthash{ $totname } ) ) {
+        push @{ $totlisthash{ $totname  } }, $tax{ $tax_object->taxname };
+      }else{
+        $totlisthash{ $totname } = [ $tot, $tax{ $tax_object->taxname } ];
+      }
+    }
+  }
 
-  } #foreach my $cust_pkg
+  warn "having a look at taxed taxes...\n" if $DEBUG > 2;
+  foreach my $tax ( keys %totlisthash ) {
+    my $tax_object = shift @{ $totlisthash{$tax} };
+    warn "found previously found taxed tax ". $tax_object->taxname. "\n"
+      if $DEBUG > 2;
+    my $listref_or_error = $tax_object->taxline( @{ $totlisthash{$tax} } );
+    unless (ref($listref_or_error)) {
+      $dbh->rollback if $oldAutoCommit;
+      return $listref_or_error;
+    }
 
-  unless ( $cust_bill->cust_bill_pkg ) {
-    $cust_bill->delete; #don't create an invoice w/o line items
-    $dbh->commit or die $dbh->errstr if $oldAutoCommit;
-    return '';
+    warn "adding taxed tax amount ". $listref_or_error->[1].
+         " as ". $tax_object->taxname. "\n"
+      if $DEBUG;
+    $tax{ $tax_object->taxname } += $listref_or_error->[1];
   }
+  
+  #consolidate and create tax line items
+  warn "consolidating and generating...\n" if $DEBUG > 2;
+  foreach my $taxname ( keys %taxname ) {
+    my $tax = 0;
+    my %seen = ();
+    warn "adding $taxname\n" if $DEBUG > 1;
+    foreach my $taxitem ( @{ $taxname{$taxname} } ) {
+      $tax += $tax{$taxitem} unless $seen{$taxitem};
+      warn "adding $tax{$taxitem}\n" if $DEBUG > 1;
+    }
+    next unless $tax;
 
-  my $charged = sprintf( "%.2f", $total_setup + $total_recur );
-
-  foreach my $taxname ( grep { $tax{$_} > 0 } keys %tax ) {
-    my $tax = sprintf("%.2f", $tax{$taxname} );
-    $charged = sprintf( "%.2f", $charged+$tax );
+    $tax = sprintf('%.2f', $tax );
+    $total_setup = sprintf('%.2f', $total_setup+$tax );
   
-    my $cust_bill_pkg = new FS::cust_bill_pkg ({
-      'invnum'   => $invnum,
+    push @cust_bill_pkg, new FS::cust_bill_pkg {
       'pkgnum'   => 0,
       'setup'    => $tax,
       'recur'    => 0,
       'sdate'    => '',
       'edate'    => '',
       'itemdesc' => $taxname,
-    });
-    $error = $cust_bill_pkg->insert;
-    if ( $error ) {
-      $dbh->rollback if $oldAutoCommit;
-      return "can't create invoice line item for invoice #$invnum: $error";
-    }
-    $total_setup += $tax;
+    };
 
   }
 
-  $cust_bill->charged( sprintf( "%.2f", $total_setup + $total_recur ) );
-  $error = $cust_bill->replace;
+  my $charged = sprintf('%.2f', $total_setup + $total_recur );
+
+  #create the new invoice
+  my $cust_bill = new FS::cust_bill ( {
+    'custnum' => $self->custnum,
+    '_date'   => ( $options{'invoice_time'} || $time ),
+    'charged' => $charged,
+  } );
+  my $error = $cust_bill->insert;
   if ( $error ) {
     $dbh->rollback if $oldAutoCommit;
-    return "can't update charged for invoice #$invnum: $error";
+    return "can't create invoice for customer #". $self->custnum. ": $error";
   }
 
-  foreach my $hook ( @precommit_hooks ) { 
-    eval {
-      &{$hook}; #($self) ?
-    };
+  foreach my $cust_bill_pkg ( @cust_bill_pkg ) {
+    $cust_bill_pkg->invnum($cust_bill->invnum); 
+    my $error = $cust_bill_pkg->insert;
+    if ( $error ) {
+      $dbh->rollback if $oldAutoCommit;
+      return "can't create invoice line item: $error";
+    }
+  }
+    
+
+  foreach my $hook ( @precommit_hooks ) { 
+    eval {
+      &{$hook}; #($self) ?
+    };
     if ( $@ ) {
       $dbh->rollback if $oldAutoCommit;
       return "$@ running precommit hook $hook\n";
@@ -2239,17 +2288,446 @@ sub bill {
   ''; #no error
 }
 
+
+sub _make_lines {
+  my ($self, %params) = @_;
+
+  my $part_pkg = $params{part_pkg} or die "no part_pkg specified";
+  my $cust_pkg = $params{cust_pkg} or die "no cust_pkg specified";
+  my $precommit_hooks = $params{precommit_hooks} or die "no package specified";
+  my $cust_bill_pkgs = $params{line_items} or die "no line buffer specified";
+  my $appended_cust_bill_pkg = $params{appended_line_items}
+    or die "no appended line buffer specified";
+  my $total_setup = $params{setup} or die "no setup accumulator specified";
+  my $total_recur = $params{recur} or die "no recur accumulator specified";
+  my $taxlisthash = $params{tax_matrix} or die "no tax accumulator specified";
+  my $time = $params{'time'} or die "no time specified";
+  my (%options) = %{$params{options}};  #hmmm  only for 'resetup'
+
+  my $dbh = dbh;
+  my $real_pkgpart = $cust_pkg->pkgpart;
+  my %hash = $cust_pkg->hash;
+  my $old_cust_pkg = new FS::cust_pkg \%hash;
+
+  my @details = ();
+
+  my $lineitems = 0;
+
+  $cust_pkg->pkgpart($part_pkg->pkgpart);
+
+  ###
+  # bill setup
+  ###
+
+  my $setup = 0;
+  my $unitsetup = 0;
+  if ( ! $cust_pkg->setup &&
+       (
+         ( $conf->exists('disable_setup_suspended_pkgs') &&
+          ! $cust_pkg->getfield('susp')
+        ) || ! $conf->exists('disable_setup_suspended_pkgs')
+       )
+    || $options{'resetup'}
+  ) {
+    
+    warn "    bill setup\n" if $DEBUG > 1;
+    $lineitems++;
+
+    $setup = eval { $cust_pkg->calc_setup( $time, \@details ) };
+    return "$@ running calc_setup for $cust_pkg\n"
+      if $@;
+
+    $unitsetup = $cust_pkg->part_pkg->unit_setup || $setup; #XXX uuh
+
+    $cust_pkg->setfield('setup', $time)
+      unless $cust_pkg->setup;
+          #do need it, but it won't get written to the db
+          #|| $cust_pkg->pkgpart != $real_pkgpart;
+
+  }
+
+  ###
+  # bill recurring fee
+  ### 
+
+  #XXX unit stuff here too
+  my $recur = 0;
+  my $unitrecur = 0;
+  my $sdate;
+  if ( $part_pkg->getfield('freq') ne '0' &&
+       ! $cust_pkg->getfield('susp') &&
+       ( $cust_pkg->getfield('bill') || 0 ) <= $time
+  ) {
+
+    # XXX should this be a package event?  probably.  events are called
+    # at collection time at the moment, though...
+    $part_pkg->reset_usage($cust_pkg, 'debug'=>$DEBUG)
+      if $part_pkg->can('reset_usage');
+      #don't want to reset usage just cause we want a line item??
+      #&& $part_pkg->pkgpart == $real_pkgpart;
+
+    warn "    bill recur\n" if $DEBUG > 1;
+    $lineitems++;
+
+    # XXX shared with $recur_prog
+    $sdate = $cust_pkg->bill || $cust_pkg->setup || $time;
+
+    #over two params!  lets at least switch to a hashref for the rest...
+    my %param = ( 'precommit_hooks' => $precommit_hooks, );
+
+    $recur = eval { $cust_pkg->calc_recur( \$sdate, \@details, \%param ) };
+    return "$@ running calc_recur for $cust_pkg\n"
+      if ( $@ );
+
+  
+    #change this bit to use Date::Manip? CAREFUL with timezones (see
+    # mailing list archive)
+    my ($sec,$min,$hour,$mday,$mon,$year) =
+      (localtime($sdate) )[0,1,2,3,4,5];
+    
+    #pro-rating magic - if $recur_prog fiddles $sdate, want to use that
+    # only for figuring next bill date, nothing else, so, reset $sdate again
+    # here
+    $sdate = $cust_pkg->bill || $cust_pkg->setup || $time;
+    $cust_pkg->last_bill($sdate);
+    
+    if ( $part_pkg->freq =~ /^\d+$/ ) {
+      $mon += $part_pkg->freq;
+      until ( $mon < 12 ) { $mon -= 12; $year++; }
+    } elsif ( $part_pkg->freq =~ /^(\d+)w$/ ) {
+      my $weeks = $1;
+      $mday += $weeks * 7;
+    } elsif ( $part_pkg->freq =~ /^(\d+)d$/ ) {
+      my $days = $1;
+      $mday += $days;
+    } elsif ( $part_pkg->freq =~ /^(\d+)h$/ ) {
+      my $hours = $1;
+      $hour += $hours;
+    } else {
+      return "unparsable frequency: ". $part_pkg->freq;
+    }
+    $cust_pkg->setfield('bill',
+      timelocal_nocheck($sec,$min,$hour,$mday,$mon,$year));
+
+  }
+
+  warn "\$setup is undefined" unless defined($setup);
+  warn "\$recur is undefined" unless defined($recur);
+  warn "\$cust_pkg->bill is undefined" unless defined($cust_pkg->bill);
+  
+  ###
+  # If there's line items, create em cust_bill_pkg records
+  # If $cust_pkg has been modified, update it (if we're a real pkgpart)
+  ###
+
+  if ( $lineitems ) {
+
+    if ( $cust_pkg->modified && $cust_pkg->pkgpart == $real_pkgpart ) {
+      # hmm.. and if just the options are modified in some weird price plan?
+  
+      warn "  package ". $cust_pkg->pkgnum. " modified; updating\n"
+        if $DEBUG >1;
+  
+      my $error = $cust_pkg->replace( $old_cust_pkg,
+                                      'options' => { $cust_pkg->options },
+                                    );
+      return "Error modifying pkgnum ". $cust_pkg->pkgnum. ": $error"
+        if $error; #just in case
+    }
+  
+    $setup = sprintf( "%.2f", $setup );
+    $recur = sprintf( "%.2f", $recur );
+    if ( $setup < 0 && ! $conf->exists('allow_negative_charges') ) {
+      return "negative setup $setup for pkgnum ". $cust_pkg->pkgnum;
+    }
+    if ( $recur < 0 && ! $conf->exists('allow_negative_charges') ) {
+      return "negative recur $recur for pkgnum ". $cust_pkg->pkgnum;
+    }
+
+    if ( $setup != 0 || $recur != 0 ) {
+
+      warn "    charges (setup=$setup, recur=$recur); adding line items\n"
+        if $DEBUG > 1;
+      my $cust_bill_pkg = new FS::cust_bill_pkg {
+        'pkgnum'    => $cust_pkg->pkgnum,
+        'setup'     => $setup,
+        'unitsetup' => $unitsetup,
+        'recur'     => $recur,
+        'unitrecur' => $unitrecur,
+        'quantity'  => $cust_pkg->quantity,
+        'sdate'     => $sdate,
+        'edate'     => $cust_pkg->bill,
+        'details'   => \@details,
+      };
+      $cust_bill_pkg->pkgpart_override($part_pkg->pkgpart)
+        unless $part_pkg->pkgpart == $real_pkgpart;
+
+      $$total_setup += $setup;
+      $$total_recur += $recur;
+
+      ###
+      # handle taxes
+      ###
+
+      unless ( $self->tax =~ /Y/i || $self->payby eq 'COMP' ) {
+
+        #some garbage disappears on cust_bill_pkg refactor
+        my $err_or_cust_bill_pkg =
+          $self->_handle_taxes($part_pkg, $taxlisthash, $cust_bill_pkg);
+
+        return $err_or_cust_bill_pkg
+          unless ( ref($err_or_cust_bill_pkg) );
+
+        push @$cust_bill_pkgs, @$err_or_cust_bill_pkg;
+
+      } #unless $self->tax =~ /Y/i || $self->payby eq 'COMP'
+
+    } #if $setup != 0 || $recur != 0
+      
+  } #if $line_items
+
+  if ( $part_pkg->can('append_cust_bill_pkgs') ) {
+    my %param = ( 'precommit_hooks' => $precommit_hooks, );
+    my ($more_cust_bill_pkgs) =
+      eval { $part_pkg->append_cust_bill_pkgs( $cust_pkg, \$sdate, \%param ) };
+
+    return "$@ running append_cust_bill_pkgs for $cust_pkg\n"
+      if ( $@ );
+    return "$more_cust_bill_pkgs"
+      unless ( ref($more_cust_bill_pkgs) );
+
+    foreach my $cust_bill_pkg ( @{$more_cust_bill_pkgs} ) {
+
+      $cust_bill_pkg->pkgpart_override($part_pkg->pkgpart)
+        unless $part_pkg->pkgpart == $real_pkgpart;
+
+      unless ($cust_bill_pkg->duplicate) {
+        $$total_setup += $cust_bill_pkg->setup;
+        $$total_recur += $cust_bill_pkg->recur;
+
+        ###
+        # handle taxes
+        ###
+
+        unless ( $self->tax =~ /Y/i || $self->payby eq 'COMP' ) {
+
+          #some garbage disappears on cust_bill_pkg refactor
+          my $err_or_cust_bill_pkg =
+            $self->_handle_taxes($part_pkg, $taxlisthash, $cust_bill_pkg);
+
+          return $err_or_cust_bill_pkg
+            unless ( ref($err_or_cust_bill_pkg) );
+
+          push @$appended_cust_bill_pkg, @$err_or_cust_bill_pkg;
+
+        } #unless $self->tax =~ /Y/i || $self->payby eq 'COMP'
+      }
+    }
+  }
+
+}
+
+sub _handle_taxes {
+  my $self = shift;
+  my $part_pkg = shift;
+  my $taxlisthash = shift;
+  my $cust_bill_pkg = shift;
+
+  my %cust_bill_pkg = ();
+  my %taxes = ();
+    
+  my $prefix = 
+    ( $conf->exists('tax-ship_address') && length($self->ship_last) )
+    ? 'ship_'
+    : '';
+
+  my @classes;
+  #push @classes, $cust_bill_pkg->usage_classes if $cust_bill_pkg->type eq 'U';
+  push @classes, $cust_bill_pkg->usage_classes if $cust_bill_pkg->usage;
+  push @classes, 'setup' if $cust_bill_pkg->setup;
+  push @classes, 'recur' if $cust_bill_pkg->recur;
+
+  if ( $conf->exists('enable_taxproducts')
+       && (scalar($part_pkg->part_pkg_taxoverride) || $part_pkg->has_taxproduct)
+     )
+  { 
+
+    foreach my $class (@classes) {
+      my $err_or_ref = $self->_gather_taxes( $part_pkg, $class, $prefix );
+      return $err_or_ref unless ref($err_or_ref);
+      $taxes{$class} = $err_or_ref;
+    }
+
+    unless (exists $taxes{''}) {
+      my $err_or_ref = $self->_gather_taxes( $part_pkg, '', $prefix );
+      return $err_or_ref unless ref($err_or_ref);
+      $taxes{''} = $err_or_ref;
+    }
+
+  }else{
+
+    my %taxhash = map { $_ => $self->get("$prefix$_") }
+                      qw( state county country );
+
+    $taxhash{'taxclass'} = $part_pkg->taxclass;
+
+    my @taxes = qsearch( 'cust_main_county', \%taxhash );
+
+    unless ( @taxes ) {
+      $taxhash{'taxclass'} = '';
+      @taxes =  qsearch( 'cust_main_county', \%taxhash );
+    }
+
+    #one more try at a whole-country tax rate
+    unless ( @taxes ) {
+      $taxhash{$_} = '' foreach qw( state county );
+      @taxes =  qsearch( 'cust_main_county', \%taxhash );
+    }
+
+    $taxes{''} = [ @taxes ];
+    $taxes{'setup'} = [ @taxes ];
+    $taxes{'recur'} = [ @taxes ];
+    $taxes{$_} = [ @taxes ] foreach (@classes);
+
+    # maybe eliminate this entirely, along with all the 0% records
+    unless ( @taxes ) {
+      return
+        "fatal: can't find tax rate for state/county/country/taxclass ".
+        join('/', ( map $self->get("$prefix$_"),
+                        qw(state county country)
+                  ),
+                  $part_pkg->taxclass ). "\n";
+    }
+
+  } #if $conf->exists('enable_taxproducts') 
+
+  # XXX all this goes away with cust_bill_pay refactor
+
+  $cust_bill_pkg{setup} = $cust_bill_pkg if $cust_bill_pkg->setup;
+  $cust_bill_pkg{recur} = $cust_bill_pkg if $cust_bill_pkg->recur;
+    
+  #split setup and recur
+  if ($cust_bill_pkg->setup && $cust_bill_pkg->recur) {
+    my $cust_bill_pkg_recur = new FS::cust_bill_pkg { $cust_bill_pkg->hash };
+    $cust_bill_pkg->set('details', []);
+    $cust_bill_pkg->recur(0);
+    $cust_bill_pkg->unitrecur(0);
+    $cust_bill_pkg->type('');
+    $cust_bill_pkg_recur->setup(0);
+    $cust_bill_pkg_recur->unitsetup(0);
+    $cust_bill_pkg{recur} = $cust_bill_pkg_recur;
+  }
+
+  #split usage from recur
+  my $usage = sprintf( "%.2f", $cust_bill_pkg{recur}->usage );
+  warn "usage is $usage\n" if $DEBUG;
+  if ($usage) {
+    my $cust_bill_pkg_usage =
+        new FS::cust_bill_pkg { $cust_bill_pkg{recur}->hash };
+    $cust_bill_pkg_usage->recur( $usage );
+    $cust_bill_pkg_usage->type( 'U' );
+    my $recur = sprintf( "%.2f", $cust_bill_pkg{recur}->recur - $usage );
+    $cust_bill_pkg{recur}->recur( $recur );
+    $cust_bill_pkg{recur}->type( '' );
+    $cust_bill_pkg{recur}->set('details', []);
+    $cust_bill_pkg{''} = $cust_bill_pkg_usage;
+  }
+
+  #subdivide usage by usage_class
+  if (exists($cust_bill_pkg{''})) {
+    foreach my $class (grep {$_ && $_ ne 'setup' && $_ ne 'recur' } @classes) {
+      my $usage = sprintf( "%.2f", $cust_bill_pkg{''}->usage($class) );
+      my $cust_bill_pkg_usage =
+          new FS::cust_bill_pkg { $cust_bill_pkg{''}->hash };
+      $cust_bill_pkg_usage->recur( $usage );
+      $cust_bill_pkg_usage->set('details', []);
+      my $classless = sprintf( "%.2f", $cust_bill_pkg{''}->recur - $usage );
+      $cust_bill_pkg{''}->recur( $classless );
+      $cust_bill_pkg{$class} = $cust_bill_pkg_usage;
+    }
+    delete $cust_bill_pkg{''} unless $cust_bill_pkg{''}->recur;
+  }
+
+  foreach my $key (keys %cust_bill_pkg) {
+    my @taxes = @{ $taxes{$key} };
+    my $cust_bill_pkg = $cust_bill_pkg{$key};
+
+    foreach my $tax ( @taxes ) {
+      my $taxname = ref( $tax ). ' '. $tax->taxnum;
+      if ( exists( $taxlisthash->{ $taxname } ) ) {
+        push @{ $taxlisthash->{ $taxname  } }, $cust_bill_pkg;
+      }else{
+        $taxlisthash->{ $taxname } = [ $tax, $cust_bill_pkg ];
+      }
+    }
+  }
+
+  # sort setup,recur,'', and the rest numeric && return
+  my @result = map { $cust_bill_pkg{$_} }
+               sort { my $ad = ($a=~/^\d+$/); my $bd = ($b=~/^\d+$/);
+                      ( $ad cmp $bd ) || ( $ad ? $a<=>$b : $b cmp $a )
+                    }
+               keys %cust_bill_pkg;
+
+  \@result;
+}
+
+sub _gather_taxes {
+  my $self = shift;
+  my $part_pkg = shift;
+  my $class = shift;
+  my $prefix = shift;
+
+  my @taxes = ();
+  my $geocode = $self->geocode('cch');
+
+  my @taxclassnums = map { $_->taxclassnum }
+                     $part_pkg->part_pkg_taxoverride($class);
+
+  unless (@taxclassnums) {
+    @taxclassnums = map { $_->taxclassnum }
+                    $part_pkg->part_pkg_taxrate('cch', $geocode, $class);
+  }
+  warn "Found taxclassnum values of ". join(',', @taxclassnums)
+    if $DEBUG;
+
+  my $extra_sql =
+    "AND (".
+    join(' OR ', map { "taxclassnum = $_" } @taxclassnums ). ")";
+
+  @taxes = qsearch({ 'table' => 'tax_rate',
+                     'hashref' => { 'geocode' => $geocode, },
+                     'extra_sql' => $extra_sql,
+                  })
+    if scalar(@taxclassnums);
+
+  # maybe eliminate this entirely, along with all the 0% records
+  unless ( @taxes ) {
+    return 
+      "fatal: can't find tax rate for zip/taxproduct/pkgpart ".
+      join('/', ( map $self->get("$prefix$_"),
+                      qw(zip)
+                ),
+                $part_pkg->taxproduct_description,
+                $part_pkg->pkgpart ). "\n";
+  }
+
+  warn "Found taxes ".
+       join(',', map{ ref($_). " ". $_->get($_->primary_key) } @taxes). "\n" 
+   if $DEBUG;
+
+  [ @taxes ];
+
+}
+
 =item collect OPTIONS
 
 (Attempt to) collect money for this customer's outstanding invoices (see
 L<FS::cust_bill>).  Usually used after the bill method.
 
-Depending on the value of `payby', this may print or email an invoice (I<BILL>,
-I<DCRD>, or I<DCHK>), charge a credit card (I<CARD>), charge via electronic
-check/ACH (I<CHEK>), or just add any necessary (pseudo-)payment (I<COMP>).
-
-Most actions are now triggered by invoice events; see L<FS::part_bill_event>
-and the invoice events web interface.
+Actions are now triggered by billing events; see L<FS::part_event> and the
+billing events web interface.  Old-style invoice events (see
+L<FS::part_bill_event>) have been deprecated.
 
 If there is an error, returns the error, otherwise returns false.
 
@@ -2257,19 +2735,34 @@ Options are passed as name-value pairs.
 
 Currently available options are:
 
-invoice_time - Use this time when deciding when to print invoices and
-late notices on those invoices.  The default is now.  It is specified as a UNIX timestamp; see L<perlfunc/"time">).  Also see L<Time::Local> and L<Date::Parse>
-for conversion functions.
+=over 4
+
+=item invoice_time
+
+Use this time when deciding when to print invoices and late notices on those invoices.  The default is now.  It is specified as a UNIX timestamp; see L<perlfunc/"time">).  Also see L<Time::Local> and L<Date::Parse> for conversion functions.
+
+=item retry
+
+Retry card/echeck/LEC transactions even when not scheduled by invoice events.
+
+=item quiet
 
-retry - Retry card/echeck/LEC transactions even when not scheduled by invoice
-events.
+set true to surpress email card/ACH decline notices.
 
-quiet - set true to surpress email card/ACH decline notices.
+=item check_freq
 
-freq - "1d" for the traditional, daily events (the default), or "1m" for the
-new monthly events
+"1d" for the traditional, daily events (the default), or "1m" for the new monthly events (part_event.check_freq)
 
-payby - allows for one time override of normal customer billing method
+=item payby
+
+allows for one time override of normal customer billing method
+
+=item debug
+
+Debugging level.  Default is 0 (no debugging), or can be set to 1 (passed-in options), 2 (traces progress), 3 (more information), or 4 (include full search queries)
+
+
+=back
 
 =cut
 
@@ -2291,12 +2784,9 @@ sub collect {
 
   $self->select_for_update; #mutex
 
-  my $balance = $self->balance;
-  warn "$me collect customer ". $self->custnum. ": balance $balance\n"
-    if $DEBUG;
-  unless ( $balance > 0 ) { #redundant?????
-    $dbh->rollback if $oldAutoCommit; #hmm
-    return '';
+  if ( $DEBUG ) {
+    my $balance = $self->balance;
+    warn "$me collect customer ". $self->custnum. ": balance $balance\n"
   }
 
   if ( exists($options{'retry_card'}) ) {
@@ -2311,45 +2801,48 @@ sub collect {
     }
   }
 
-  my $extra_sql = '';
-  if ( defined $options{'freq'} && $options{'freq'} eq '1m' ) {
-    $extra_sql = " AND freq = '1m' ";
-  } else {
-    $extra_sql = " AND ( freq = '1d' OR freq IS NULL OR freq = '' ) ";
-  }
-
-  foreach my $cust_bill ( $self->open_cust_bill ) {
-
-    # don't try to charge for the same invoice if it's already in a batch
-    #next if qsearchs( 'cust_pay_batch', { 'invnum' => $cust_bill->invnum } );
+  # false laziness w/pay_batch::import_results
 
-    last if $self->balance <= 0;
-
-    warn "  invnum ". $cust_bill->invnum. " (owed ". $cust_bill->owed. ")\n"
-      if $DEBUG > 1;
+  my $due_cust_event = $self->due_cust_event(
+    'debug'      => ( $options{'debug'} || 0 ),
+    'time'       => $invoice_time,
+    'check_freq' => $options{'check_freq'},
+  );
+  unless( ref($due_cust_event) ) {
+    $dbh->rollback if $oldAutoCommit;
+    return $due_cust_event;
+  }
 
-    foreach my $part_bill_event ( due_events ( $cust_bill,
-                                               exists($options{'payby'}) 
-					         ? $options{'payby'}
-						 : $self->payby,
-					       $invoice_time,
-					       $extra_sql ) ) {
+  foreach my $cust_event ( @$due_cust_event ) {
 
-      last if $cust_bill->owed <= 0  # don't run subsequent events if owed<=0
-           || $self->balance   <= 0; # or if balance<=0
+    #XXX lock event
+    
+    #re-eval event conditions (a previous event could have changed things)
+    unless ( $cust_event->test_conditions( 'time' => $invoice_time ) ) {
+      #don't leave stray "new/locked" records around
+      my $error = $cust_event->delete;
+      if ( $error ) {
+        #gah, even with transactions
+        $dbh->commit if $oldAutoCommit; #well.
+        return $error;
+      }
+      next;
+    }
 
-      {
-        local $realtime_bop_decline_quiet = 1 if $options{'quiet'};
-        warn "  do_event " .  $cust_bill . " ". (%options) .  "\n"
-          if $DEBUG > 1;
+    {
+      local $realtime_bop_decline_quiet = 1 if $options{'quiet'};
+      warn "  running cust_event ". $cust_event->eventnum. "\n"
+        if $DEBUG > 1;
 
-        if (my $error = $part_bill_event->do_event($cust_bill, %options)) {
+      
+      #if ( my $error = $cust_event->do_event(%options) ) { #XXX %options?
+      if ( my $error = $cust_event->do_event() ) {
+        #XXX wtf is this?  figure out a proper dealio with return value
+        #from do_event
 	  # gah, even with transactions.
 	  $dbh->commit if $oldAutoCommit; #well.
 	  return $error;
 	}
-      }
-
     }
 
   }
@@ -2359,21 +2852,59 @@ sub collect {
 
 }
 
-=item retry_realtime
+=item due_cust_event [ HASHREF | OPTION => VALUE ... ]
 
-Schedules realtime / batch  credit card / electronic check / LEC billing
-events for for retry.  Useful if card information has changed or manual
-retry is desired.  The 'collect' method must be called to actually retry
-the transaction.
+Inserts database records for and returns an ordered listref of new events due
+for this customer, as FS::cust_event objects (see L<FS::cust_event>).  If no
+events are due, an empty listref is returned.  If there is an error, returns a
+scalar error message.
+
+To actually run the events, call each event's test_condition method, and if
+still true, call the event's do_event method.
+
+Options are passed as a hashref or as a list of name-value pairs.  Available
+options are:
+
+=over 4
+
+=item check_freq
+
+Search only for events of this check frequency (how often events of this type are checked); currently "1d" (daily, the default) and "1m" (monthly) are recognized.
 
-Implementation details: For each of this customer's open invoices, changes
-the status of the first "done" (with statustext error) realtime processing
-event to "failed".
+=item time
+
+"Current time" for the events.
+
+=item debug
+
+Debugging level.  Default is 0 (no debugging), or can be set to 1 (passed-in options), 2 (traces progress), 3 (more information), or 4 (include full search queries)
+
+=item eventtable
+
+Only return events for the specified eventtable (by default, events of all eventtables are returned)
+
+=item objects
+
+Explicitly pass the objects to be tested (typically used with eventtable).
+
+=back
 
 =cut
 
-sub retry_realtime {
+sub due_cust_event {
   my $self = shift;
+  my %opt = ref($_[0]) ? %{ $_[0] } : @_;
+
+  #???
+  #my $DEBUG = $opt{'debug'}
+  local($DEBUG) = $opt{'debug'}
+    if defined($opt{'debug'}) && $opt{'debug'} > $DEBUG;
+
+  warn "$me due_cust_event called with options ".
+       join(', ', map { "$_: $opt{$_}" } keys %opt). "\n"
+    if $DEBUG;
+
+  $opt{'time'} ||= time;
 
   local $SIG{HUP} = 'IGNORE';
   local $SIG{INT} = 'IGNORE';
@@ -2386,83 +2917,306 @@ sub retry_realtime {
   local $FS::UID::AutoCommit = 0;
   my $dbh = dbh;
 
-  foreach my $cust_bill (
-    grep { $_->cust_bill_event }
-      $self->open_cust_bill
-  ) {
-    my @cust_bill_event =
-      sort { $a->part_bill_event->seconds <=> $b->part_bill_event->seconds }
-        grep {
-               #$_->part_bill_event->plan eq 'realtime-card'
-               $_->part_bill_event->eventcode =~
-                   /\$cust_bill\->(batch|realtime)_(card|ach|lec)/
-                 && $_->status eq 'done'
-                 && $_->statustext
-             }
-          $cust_bill->cust_bill_event;
-    next unless @cust_bill_event;
-    my $error = $cust_bill_event[0]->retry;
-    if ( $error ) {
-      $dbh->rollback if $oldAutoCommit;
-      return "error scheduling invoice event for retry: $error";
-    }
+  $self->select_for_update; #mutex
 
-  }
+  ###
+  # 1: find possible events (initial search)
+  ###
+  
+  my @cust_event = ();
 
-  $dbh->commit or die $dbh->errstr if $oldAutoCommit;
-  '';
+  my @eventtable = $opt{'eventtable'}
+                     ? ( $opt{'eventtable'} )
+                     : FS::part_event->eventtables_runorder;
 
-}
+  foreach my $eventtable ( @eventtable ) {
 
-=item realtime_bop METHOD AMOUNT [ OPTION => VALUE ... ]
+    my @objects;
+    if ( $opt{'objects'} ) {
 
-Runs a realtime credit card, ACH (electronic check) or phone bill transaction
-via a Business::OnlinePayment realtime gateway.  See
-L<http://420.am/business-onlinepayment> for supported gateways.
+      @objects = @{ $opt{'objects'} };
 
-Available methods are: I<CC>, I<ECHECK> and I<LEC>
+    } else {
 
-Available options are: I<description>, I<invnum>, I<quiet>
+      #my @objects = $self->eventtable(); # sub cust_main { @{ [ $self ] }; }
+      @objects = ( $eventtable eq 'cust_main' )
+                   ? ( $self )
+                   : ( $self->$eventtable() );
 
-The additional options I<payname>, I<address1>, I<address2>, I<city>, I<state>,
-I<zip>, I<payinfo> and I<paydate> are also available.  Any of these options,
-if set, will override the value from the customer record.
+    }
 
-I<description> is a free-text field passed to the gateway.  It defaults to
-"Internet services".
+    my @e_cust_event = ();
 
-If an I<invnum> is specified, this payment (if successful) is applied to the
-specified invoice.  If you don't specify an I<invnum> you might want to
-call the B<apply_payments> method.
+    my $cross = "CROSS JOIN $eventtable";
+    $cross .= ' LEFT JOIN cust_main USING ( custnum )'
+      unless $eventtable eq 'cust_main';
 
-I<quiet> can be set true to surpress email decline notices.
+    foreach my $object ( @objects ) {
 
-(moved from cust_bill) (probably should get realtime_{card,ach,lec} here too)
+      #this first search uses the condition_sql magic for optimization.
+      #the more possible events we can eliminate in this step the better
 
-=cut
+      my $cross_where = '';
+      my $pkey = $object->primary_key;
+      $cross_where = "$eventtable.$pkey = ". $object->$pkey();
 
-sub realtime_bop {
-  my( $self, $method, $amount, %options ) = @_;
-  if ( $DEBUG ) {
-    warn "$me realtime_bop: $method $amount\n";
-    warn "  $_ => $options{$_}\n" foreach keys %options;
-  }
+      my $join = FS::part_event_condition->join_conditions_sql( $eventtable );
+      my $extra_sql =
+        FS::part_event_condition->where_conditions_sql( $eventtable,
+                                                        'time'=>$opt{'time'}
+                                                      );
+      my $order = FS::part_event_condition->order_conditions_sql( $eventtable );
 
-  $options{'description'} ||= 'Internet services';
+      $extra_sql = "AND $extra_sql" if $extra_sql;
 
-  eval "use Business::OnlinePayment";  
-  die $@ if $@;
+      #here is the agent virtualization
+      $extra_sql .= " AND (    part_event.agentnum IS NULL
+                            OR part_event.agentnum = ". $self->agentnum. ' )';
 
-  my $payinfo = exists($options{'payinfo'})
-                  ? $options{'payinfo'}
-                  : $self->payinfo;
+      $extra_sql .= " $order";
 
-  ###
-  # select a gateway
-  ###
+      warn "searching for events for $eventtable ". $object->$pkey. "\n"
+        if $opt{'debug'} > 2;
+      my @part_event = qsearch( {
+        'debug'     => ( $opt{'debug'} > 3 ? 1 : 0 ),
+        'select'    => 'part_event.*',
+        'table'     => 'part_event',
+        'addl_from' => "$cross $join",
+        'hashref'   => { 'check_freq' => ( $opt{'check_freq'} || '1d' ),
+                         'eventtable' => $eventtable,
+                         'disabled'   => '',
+                       },
+        'extra_sql' => "AND $cross_where $extra_sql",
+      } );
 
-  my $taxclass = '';
-  if ( $options{'invnum'} ) {
+      if ( $DEBUG > 2 ) {
+        my $pkey = $object->primary_key;
+        warn "      ". scalar(@part_event).
+             " possible events found for $eventtable ". $object->$pkey(). "\n";
+      }
+
+      push @e_cust_event, map { $_->new_cust_event($object) } @part_event;
+
+    }
+
+    warn "    ". scalar(@e_cust_event).
+         " subtotal possible cust events found for $eventtable\n"
+      if $DEBUG > 1;
+
+    push @cust_event, @e_cust_event;
+
+  }
+
+  warn "  ". scalar(@cust_event).
+       " total possible cust events found in initial search\n"
+    if $DEBUG; # > 1;
+
+  ##
+  # 2: test conditions
+  ##
+  
+  my %unsat = ();
+
+  @cust_event = grep $_->test_conditions( 'time'          => $opt{'time'},
+                                          'stats_hashref' => \%unsat ),
+                     @cust_event;
+
+  warn "  ". scalar(@cust_event). " cust events left satisfying conditions\n"
+    if $DEBUG; # > 1;
+
+  warn "    invalid conditions not eliminated with condition_sql:\n".
+       join('', map "      $_: ".$unsat{$_}."\n", keys %unsat )
+    if $DEBUG; # > 1;
+
+  ##
+  # 3: insert
+  ##
+
+  foreach my $cust_event ( @cust_event ) {
+
+    my $error = $cust_event->insert();
+    if ( $error ) {
+      $dbh->rollback if $oldAutoCommit;
+      return $error;
+    }
+                                       
+  }
+
+  $dbh->commit or die $dbh->errstr if $oldAutoCommit;
+
+  ##
+  # 4: return
+  ##
+
+  warn "  returning events: ". Dumper(@cust_event). "\n"
+    if $DEBUG > 2;
+
+  \@cust_event;
+
+}
+
+=item retry_realtime
+
+Schedules realtime / batch  credit card / electronic check / LEC billing
+events for for retry.  Useful if card information has changed or manual
+retry is desired.  The 'collect' method must be called to actually retry
+the transaction.
+
+Implementation details: For either this customer, or for each of this
+customer's open invoices, changes the status of the first "done" (with
+statustext error) realtime processing event to "failed".
+
+=cut
+
+sub retry_realtime {
+  my $self = shift;
+
+  local $SIG{HUP} = 'IGNORE';
+  local $SIG{INT} = 'IGNORE';
+  local $SIG{QUIT} = 'IGNORE';
+  local $SIG{TERM} = 'IGNORE';
+  local $SIG{TSTP} = 'IGNORE';
+  local $SIG{PIPE} = 'IGNORE';
+
+  my $oldAutoCommit = $FS::UID::AutoCommit;
+  local $FS::UID::AutoCommit = 0;
+  my $dbh = dbh;
+
+  #a little false laziness w/due_cust_event (not too bad, really)
+
+  my $join = FS::part_event_condition->join_conditions_sql;
+  my $order = FS::part_event_condition->order_conditions_sql;
+  my $mine = 
+  '( '
+   . join ( ' OR ' , map { 
+    "( part_event.eventtable = " . dbh->quote($_) 
+    . " AND tablenum IN( SELECT " . dbdef->table($_)->primary_key . " from $_ where custnum = " . dbh->quote( $self->custnum ) . "))" ;
+   } FS::part_event->eventtables)
+   . ') ';
+
+  #here is the agent virtualization
+  my $agent_virt = " (    part_event.agentnum IS NULL
+                       OR part_event.agentnum = ". $self->agentnum. ' )';
+
+  #XXX this shouldn't be hardcoded, actions should declare it...
+  my @realtime_events = qw(
+    cust_bill_realtime_card
+    cust_bill_realtime_check
+    cust_bill_realtime_lec
+    cust_bill_batch
+  );
+
+  my $is_realtime_event = ' ( '. join(' OR ', map "part_event.action = '$_'",
+                                                  @realtime_events
+                                     ).
+                          ' ) ';
+
+  my @cust_event = qsearchs({
+    'table'     => 'cust_event',
+    'select'    => 'cust_event.*',
+    'addl_from' => "LEFT JOIN part_event USING ( eventpart ) $join",
+    'hashref'   => { 'status' => 'done' },
+    'extra_sql' => " AND statustext IS NOT NULL AND statustext != '' ".
+                   " AND $mine AND $is_realtime_event AND $agent_virt $order" # LIMIT 1"
+  });
+
+  my %seen_invnum = ();
+  foreach my $cust_event (@cust_event) {
+
+    #max one for the customer, one for each open invoice
+    my $cust_X = $cust_event->cust_X;
+    next if $seen_invnum{ $cust_event->part_event->eventtable eq 'cust_bill'
+                          ? $cust_X->invnum
+                          : 0
+                        }++
+         or $cust_event->part_event->eventtable eq 'cust_bill'
+            && ! $cust_X->owed;
+
+    my $error = $cust_event->retry;
+    if ( $error ) {
+      $dbh->rollback if $oldAutoCommit;
+      return "error scheduling event for retry: $error";
+    }
+
+  }
+
+  $dbh->commit or die $dbh->errstr if $oldAutoCommit;
+  '';
+
+}
+
+=item realtime_bop METHOD AMOUNT [ OPTION => VALUE ... ]
+
+Runs a realtime credit card, ACH (electronic check) or phone bill transaction
+via a Business::OnlinePayment realtime gateway.  See
+L<http://420.am/business-onlinepayment> for supported gateways.
+
+Available methods are: I<CC>, I<ECHECK> and I<LEC>
+
+Available options are: I<description>, I<invnum>, I<quiet>, I<paynum_ref>, I<payunique>
+
+The additional options I<payname>, I<address1>, I<address2>, I<city>, I<state>,
+I<zip>, I<payinfo> and I<paydate> are also available.  Any of these options,
+if set, will override the value from the customer record.
+
+I<description> is a free-text field passed to the gateway.  It defaults to
+"Internet services".
+
+If an I<invnum> is specified, this payment (if successful) is applied to the
+specified invoice.  If you don't specify an I<invnum> you might want to
+call the B<apply_payments> method.
+
+I<quiet> can be set true to surpress email decline notices.
+
+I<paynum_ref> can be set to a scalar reference.  It will be filled in with the
+resulting paynum, if any.
+
+I<payunique> is a unique identifier for this payment.
+
+(moved from cust_bill) (probably should get realtime_{card,ach,lec} here too)
+
+=cut
+
+sub realtime_bop {
+  my( $self, $method, $amount, %options ) = @_;
+  if ( $DEBUG ) {
+    warn "$me realtime_bop: $method $amount\n";
+    warn "  $_ => $options{$_}\n" foreach keys %options;
+  }
+
+  $options{'description'} ||= 'Internet services';
+
+  return $self->fake_bop($method, $amount, %options) if $options{'fake'};
+
+  eval "use Business::OnlinePayment";  
+  die $@ if $@;
+
+  my $payinfo = exists($options{'payinfo'})
+                  ? $options{'payinfo'}
+                  : $self->payinfo;
+
+  my %method2payby = (
+    'CC'     => 'CARD',
+    'ECHECK' => 'CHEK',
+    'LEC'    => 'LECB',
+  );
+
+  ###
+  # check for banned credit card/ACH
+  ###
+
+  my $ban = qsearchs('banned_pay', {
+    'payby'   => $method2payby{$method},
+    'payinfo' => md5_base64($payinfo),
+  } );
+  return "Banned credit card" if $ban;
+
+  ###
+  # select a gateway
+  ###
+
+  my $taxclass = '';
+  if ( $options{'invnum'} ) {
     my $cust_bill = qsearchs('cust_bill', { 'invnum' => $options{'invnum'} } );
     die "invnum ". $options{'invnum'}. " not found" unless $cust_bill;
     my @taxclasses =
@@ -2567,6 +3321,10 @@ sub realtime_bop {
   $content{invoice_number} = $options{'invnum'}
     if exists($options{'invnum'}) && length($options{'invnum'});
 
+  $content{email_customer} = 
+    (    $conf->exists('business-onlinepayment-email_customer')
+      || $conf->exists('business-onlinepayment-email-override') );
+      
   my $paydate = '';
   if ( $method eq 'CC' ) { 
 
@@ -2580,7 +3338,7 @@ sub realtime_bop {
     my $paycvv = exists($options{'paycvv'})
                    ? $options{'paycvv'}
                    : $self->paycvv;
-    $content{cvv2} = $self->paycvv
+    $content{cvv2} = $paycvv
       if length($paycvv);
 
     my $paystart_month = exists($options{'paystart_month'})
@@ -2639,6 +3397,49 @@ sub realtime_bop {
   # run transaction(s)
   ###
 
+  my $balance = exists( $options{'balance'} )
+                  ? $options{'balance'}
+                  : $self->balance;
+
+  $self->select_for_update; #mutex ... just until we get our pending record in
+
+  #the checks here are intended to catch concurrent payments
+  #double-form-submission prevention is taken care of in cust_pay_pending::check
+
+  #check the balance
+  return "The customer's balance has changed; $method transaction aborted."
+    if $self->balance < $balance;
+    #&& $self->balance < $amount; #might as well anyway?
+
+  #also check and make sure there aren't *other* pending payments for this cust
+
+  my @pending = qsearch('cust_pay_pending', {
+    'custnum' => $self->custnum,
+    'status'  => { op=>'!=', value=>'done' } 
+  });
+  return "A payment is already being processed for this customer (".
+         join(', ', map 'paypendingnum '. $_->paypendingnum, @pending ).
+         "); $method transaction aborted."
+    if scalar(@pending);
+
+  #okay, good to go, if we're a duplicate, cust_pay_pending will kick us out
+
+  my $cust_pay_pending = new FS::cust_pay_pending {
+    'custnum'    => $self->custnum,
+    #'invnum'     => $options{'invnum'},
+    'paid'       => $amount,
+    '_date'      => '',
+    'payby'      => $method2payby{$method},
+    'payinfo'    => $payinfo,
+    'paydate'    => $paydate,
+    'status'     => 'new',
+    'gatewaynum' => ( $payment_gateway ? $payment_gateway->gatewaynum : '' ),
+  };
+  $cust_pay_pending->payunique( $options{payunique} )
+    if defined($options{payunique}) && length($options{payunique});
+  my $cpp_new_err = $cust_pay_pending->insert; #mutex lost when this is inserted
+  return $cpp_new_err if $cpp_new_err;
+
   my( $action1, $action2 ) = split(/\s*\,\s*/, $action );
 
   my $transaction = new Business::OnlinePayment( $processor, @bop_options );
@@ -2672,9 +3473,33 @@ sub realtime_bop {
     'phone'          => $self->daytime || $self->night,
     %content, #after
   );
-  $transaction->submit();
+
+  $cust_pay_pending->status('pending');
+  my $cpp_pending_err = $cust_pay_pending->replace;
+  return $cpp_pending_err if $cpp_pending_err;
+
+  #config?
+  my $BOP_TESTING = 0;
+  my $BOP_TESTING_SUCCESS = 1;
+
+  unless ( $BOP_TESTING ) {
+    $transaction->submit();
+  } else {
+    if ( $BOP_TESTING_SUCCESS ) {
+      $transaction->is_success(1);
+      $transaction->authorization('fake auth');
+    } else {
+      $transaction->is_success(0);
+      $transaction->error_message('fake failure');
+    }
+  }
 
   if ( $transaction->is_success() && $action2 ) {
+
+    $cust_pay_pending->status('authorized');
+    my $cpp_authorized_err = $cust_pay_pending->replace;
+    return $cpp_authorized_err if $cpp_authorized_err;
+
     my $auth = $transaction->authorization;
     my $ordernum = $transaction->can('order_number')
                    ? $transaction->order_number
@@ -2716,6 +3541,10 @@ sub realtime_bop {
 
   }
 
+  $cust_pay_pending->status($transaction->is_success() ? 'captured' : 'declined');
+  my $cpp_captured_err = $cust_pay_pending->replace;
+  return $cpp_captured_err if $cpp_captured_err;
+
   ###
   # remove paycvv after initial transaction
   ###
@@ -2738,12 +3567,6 @@ sub realtime_bop {
 
   if ( $transaction->is_success() ) {
 
-    my %method2payby = (
-      'CC'     => 'CARD',
-      'ECHECK' => 'CHEK',
-      'LEC'    => 'LECB',
-    );
-
     my $paybatch = '';
     if ( $payment_gateway ) { # agent override
       $paybatch = $payment_gateway->gatewaynum. '-';
@@ -2759,13 +3582,21 @@ sub realtime_bop {
        'custnum'  => $self->custnum,
        'invnum'   => $options{'invnum'},
        'paid'     => $amount,
-       '_date'     => '',
+       '_date'    => '',
        'payby'    => $method2payby{$method},
        'payinfo'  => $payinfo,
        'paybatch' => $paybatch,
        'paydate'  => $paydate,
     } );
-    $cust_pay->payunique( $options{payunique} ) if length($options{payunique});
+    #doesn't hurt to know, even though the dup check is in cust_pay_pending now
+    $cust_pay->payunique( $options{payunique} )
+      if defined($options{payunique}) && length($options{payunique});
+
+    my $oldAutoCommit = $FS::UID::AutoCommit;
+    local $FS::UID::AutoCommit = 0;
+    my $dbh = dbh;
+
+    #start a transaction, insert the cust_pay and set cust_pay_pending.status to done in a single transction
 
     my $error = $cust_pay->insert($options{'manual'} ? ( 'manual' => 1 ) : () );
 
@@ -2775,16 +3606,41 @@ sub realtime_bop {
                                       ( 'manual' => 1 ) : ()
                                     );
       if ( $error2 ) {
-        # gah, even with transactions.
-        my $e = 'WARNING: Card/ACH debited but database not updated - '.
+        # gah.  but at least we have a record of the state we had to abort in
+        # from cust_pay_pending now.
+        my $e = "WARNING: $method captured but payment not recorded - ".
                 "error inserting payment ($processor): $error2".
                 " (previously tried insert with invnum #$options{'invnum'}" .
-                ": $error )";
+                ": $error ) - pending payment saved as paypendingnum ".
+                $cust_pay_pending->paypendingnum. "\n";
         warn $e;
         return $e;
       }
     }
-    return ''; #no error
+
+    if ( $options{'paynum_ref'} ) {
+      ${ $options{'paynum_ref'} } = $cust_pay->paynum;
+    }
+
+    $cust_pay_pending->status('done');
+    $cust_pay_pending->statustext('captured');
+    my $cpp_done_err = $cust_pay_pending->replace;
+
+    if ( $cpp_done_err ) {
+
+      $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
+      my $e = "WARNING: $method captured but payment not recorded - ".
+              "error updating status for paypendingnum ".
+              $cust_pay_pending->paypendingnum. ": $cpp_done_err \n";
+      warn $e;
+      return $e;
+
+    } else {
+
+      $dbh->commit or die $dbh->errstr if $oldAutoCommit;
+      return ''; #no error
+
+    }
 
   } else {
 
@@ -2845,12 +3701,93 @@ sub realtime_bop {
         if $error;
 
     }
-  
+
+    $cust_pay_pending->status('done');
+    $cust_pay_pending->statustext("declined: $perror");
+    my $cpp_done_err = $cust_pay_pending->replace;
+    if ( $cpp_done_err ) {
+      my $e = "WARNING: $method declined but pending payment not resolved - ".
+              "error updating status for paypendingnum ".
+              $cust_pay_pending->paypendingnum. ": $cpp_done_err \n";
+      warn $e;
+      $perror = "$e ($perror)";
+    }
+
     return $perror;
   }
 
 }
 
+=item fake_bop
+
+=cut
+
+sub fake_bop {
+  my( $self, $method, $amount, %options ) = @_;
+
+  if ( $options{'fake_failure'} ) {
+     return "Error: No error; test failure requested with fake_failure";
+  }
+
+  my %method2payby = (
+    'CC'     => 'CARD',
+    'ECHECK' => 'CHEK',
+    'LEC'    => 'LECB',
+  );
+
+  #my $paybatch = '';
+  #if ( $payment_gateway ) { # agent override
+  #  $paybatch = $payment_gateway->gatewaynum. '-';
+  #}
+  #
+  #$paybatch .= "$processor:". $transaction->authorization;
+  #
+  #$paybatch .= ':'. $transaction->order_number
+  #  if $transaction->can('order_number')
+  #  && length($transaction->order_number);
+
+  my $paybatch = 'FakeProcessor:54:32';
+
+  my $cust_pay = new FS::cust_pay ( {
+     'custnum'  => $self->custnum,
+     'invnum'   => $options{'invnum'},
+     'paid'     => $amount,
+     '_date'    => '',
+     'payby'    => $method2payby{$method},
+     #'payinfo'  => $payinfo,
+     'payinfo'  => '4111111111111111',
+     'paybatch' => $paybatch,
+     #'paydate'  => $paydate,
+     'paydate'  => '2012-05-01',
+  } );
+  $cust_pay->payunique( $options{payunique} ) if length($options{payunique});
+
+  my $error = $cust_pay->insert($options{'manual'} ? ( 'manual' => 1 ) : () );
+
+  if ( $error ) {
+    $cust_pay->invnum(''); #try again with no specific invnum
+    my $error2 = $cust_pay->insert( $options{'manual'} ?
+                                    ( 'manual' => 1 ) : ()
+                                  );
+    if ( $error2 ) {
+      # gah, even with transactions.
+      my $e = 'WARNING: Card/ACH debited but database not updated - '.
+              "error inserting (fake!) payment: $error2".
+              " (previously tried insert with invnum #$options{'invnum'}" .
+              ": $error )";
+      warn $e;
+      return $e;
+    }
+  }
+
+  if ( $options{'paynum_ref'} ) {
+    ${ $options{'paynum_ref'} } = $cust_pay->paynum;
+  }
+
+  return ''; #no error
+
+}
+
 =item default_payment_gateway
 
 =cut
@@ -2864,7 +3801,7 @@ sub default_payment_gateway {
   #load up config
   my $bop_config = 'business-onlinepayment';
   $bop_config .= '-ach'
-    if $method eq 'ECHECK' && $conf->exists($bop_config. '-ach');
+    if $method =~ /^(ECHECK|CHEK)$/ && $conf->exists($bop_config. '-ach');
   my ( $processor, $login, $password, $action, @bop_options ) =
     $conf->config($bop_config);
   $action ||= 'normal authorization';
@@ -3245,6 +4182,8 @@ sub batch_card {
   local $FS::UID::AutoCommit = 0;
   my $dbh = dbh;
 
+  #this needs to handle mysql as well as Pg, like svc_acct.pm
+  #(make it into a common function if folks need to do batching with mysql)
   $dbh->do("LOCK TABLE pay_batch IN SHARE ROW EXCLUSIVE MODE")
     or return "Cannot lock pay_batch: " . $dbh->errstr;
 
@@ -3378,15 +4317,37 @@ Applies unapplied payments and credits.
 In most cases, this new method should be used in place of sequential
 apply_payments and apply_credits methods.
 
+If there is an error, returns the error, otherwise returns false.
+
 =cut
 
 sub apply_payments_and_credits {
   my $self = shift;
 
+  local $SIG{HUP} = 'IGNORE';
+  local $SIG{INT} = 'IGNORE';
+  local $SIG{QUIT} = 'IGNORE';
+  local $SIG{TERM} = 'IGNORE';
+  local $SIG{TSTP} = 'IGNORE';
+  local $SIG{PIPE} = 'IGNORE';
+
+  my $oldAutoCommit = $FS::UID::AutoCommit;
+  local $FS::UID::AutoCommit = 0;
+  my $dbh = dbh;
+
+  $self->select_for_update; #mutex
+
   foreach my $cust_bill ( $self->open_cust_bill ) {
-    $cust_bill->apply_payments_and_credits;
+    my $error = $cust_bill->apply_payments_and_credits;
+    if ( $error ) {
+      $dbh->rollback if $oldAutoCommit;
+      return "Error applying: $error";
+    }
   }
 
+  $dbh->commit or die $dbh->errstr if $oldAutoCommit;
+  ''; #no error
+
 }
 
 =item apply_credits OPTION => VALUE ...
@@ -3397,13 +4358,31 @@ chronological order if the I<order> option is set to B<newest>) and returns the
 value of any remaining unapplied credits available for refund (see
 L<FS::cust_refund>).
 
+Dies if there is an error.
+
 =cut
 
 sub apply_credits {
   my $self = shift;
   my %opt = @_;
 
-  return 0 unless $self->total_credited;
+  local $SIG{HUP} = 'IGNORE';
+  local $SIG{INT} = 'IGNORE';
+  local $SIG{QUIT} = 'IGNORE';
+  local $SIG{TERM} = 'IGNORE';
+  local $SIG{TSTP} = 'IGNORE';
+  local $SIG{PIPE} = 'IGNORE';
+
+  my $oldAutoCommit = $FS::UID::AutoCommit;
+  local $FS::UID::AutoCommit = 0;
+  my $dbh = dbh;
+
+  $self->select_for_update; #mutex
+
+  unless ( $self->total_credited ) {
+    $dbh->commit or die $dbh->errstr if $oldAutoCommit;
+    return 0;
+  }
 
   my @credits = sort { $b->_date <=> $a->_date} (grep { $_->credited > 0 }
       qsearch('cust_credit', { 'custnum' => $self->custnum } ) );
@@ -3432,13 +4411,20 @@ sub apply_credits {
       'amount'  => $amount,
     } );
     my $error = $cust_credit_bill->insert;
-    die $error if $error;
+    if ( $error ) {
+      $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
+      die $error;
+    }
     
     redo if ($cust_bill->owed > 0);
 
   }
 
-  return $self->total_credited;
+  my $total_credited = $self->total_credited;
+
+  $dbh->commit or die $dbh->errstr if $oldAutoCommit;
+
+  return $total_credited;
 }
 
 =item apply_payments
@@ -3448,17 +4434,32 @@ to outstanding invoice balances in chronological order.
 
  #and returns the value of any remaining unapplied payments.
 
+Dies if there is an error.
+
 =cut
 
 sub apply_payments {
   my $self = shift;
 
-  #return 0 unless
-
-  my @payments = sort { $b->_date <=> $a->_date } ( grep { $_->unapplied > 0 }
-      qsearch('cust_pay', { 'custnum' => $self->custnum } ) );
-
-  my @invoices = sort { $a->_date <=> $b->_date} (grep { $_->owed > 0 }
+  local $SIG{HUP} = 'IGNORE';
+  local $SIG{INT} = 'IGNORE';
+  local $SIG{QUIT} = 'IGNORE';
+  local $SIG{TERM} = 'IGNORE';
+  local $SIG{TSTP} = 'IGNORE';
+  local $SIG{PIPE} = 'IGNORE';
+
+  my $oldAutoCommit = $FS::UID::AutoCommit;
+  local $FS::UID::AutoCommit = 0;
+  my $dbh = dbh;
+
+  $self->select_for_update; #mutex
+
+  #return 0 unless
+
+  my @payments = sort { $b->_date <=> $a->_date } ( grep { $_->unapplied > 0 }
+      qsearch('cust_pay', { 'custnum' => $self->custnum } ) );
+
+  my @invoices = sort { $a->_date <=> $b->_date} (grep { $_->owed > 0 }
       qsearch('cust_bill', { 'custnum' => $self->custnum } ) );
 
   my $payment;
@@ -3482,13 +4483,20 @@ sub apply_payments {
       'amount' => $amount,
     } );
     my $error = $cust_bill_pay->insert;
-    die $error if $error;
+    if ( $error ) {
+      $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
+      die $error;
+    }
 
     redo if ( $cust_bill->owed > 0);
 
   }
 
-  return $self->total_unapplied_payments;
+  my $total_unapplied_payments = $self->total_unapplied_payments;
+
+  $dbh->commit or die $dbh->errstr if $oldAutoCommit;
+
+  return $total_unapplied_payments;
 }
 
 =item total_credited
@@ -3527,17 +4535,38 @@ sub total_unapplied_payments {
   sprintf( "%.2f", $total_unapplied );
 }
 
+=item total_unapplied_refunds
+
+Returns the total unrefunded refunds (see L<FS::cust_refund>) for this
+customer.  See L<FS::cust_refund/unapplied>.
+
+=cut
+
+sub total_unapplied_refunds {
+  my $self = shift;
+  my $total_unapplied = 0;
+  foreach my $cust_refund ( qsearch('cust_refund', {
+    'custnum' => $self->custnum,
+  } ) ) {
+    $total_unapplied += $cust_refund->unapplied;
+  }
+  sprintf( "%.2f", $total_unapplied );
+}
+
 =item balance
 
-Returns the balance for this customer (total_owed minus total_credited
-minus total_unapplied_payments).
+Returns the balance for this customer (total_owed plus total_unrefunded, minus
+total_credited minus total_unapplied_payments).
 
 =cut
 
 sub balance {
   my $self = shift;
   sprintf( "%.2f",
-    $self->total_owed - $self->total_credited - $self->total_unapplied_payments
+      $self->total_owed
+    + $self->total_unapplied_refunds
+    - $self->total_credited
+    - $self->total_unapplied_payments
   );
 }
 
@@ -3555,7 +4584,8 @@ sub balance_date {
   my $self = shift;
   my $time = shift;
   sprintf( "%.2f",
-    $self->total_owed_date($time)
+        $self->total_owed_date($time)
+      + $self->total_unapplied_refunds
       - $self->total_credited
       - $self->total_unapplied_payments
   );
@@ -3673,7 +4703,8 @@ is an error, returns the error, otherwise returns false.
 
 sub check_invoicing_list {
   my( $self, $arrayref ) = @_;
-  foreach my $address ( @{$arrayref} ) {
+
+  foreach my $address ( @$arrayref ) {
 
     if ($address eq 'FAX' and $self->getfield('fax') eq '') {
       return 'Can\'t add FAX invoice destination with a blank FAX number.';
@@ -3688,7 +4719,13 @@ sub check_invoicing_list {
                 : $cust_main_invoice->checkdest
     ;
     return $error if $error;
+
   }
+
+  return "Email address required"
+    if $conf->exists('cust_main-require_invoicing_list_email')
+    && ! grep { $_ !~ /^([A-Z]+)$/ } @$arrayref;
+
   '';
 }
 
@@ -3845,13 +4882,13 @@ otherwise returns false.
 =cut
 
 sub credit {
-  my( $self, $amount, $reason ) = @_;
+  my( $self, $amount, $reason, %options ) = @_;
   my $cust_credit = new FS::cust_credit {
     'custnum' => $self->custnum,
     'amount'  => $amount,
     'reason'  => $reason,
   };
-  $cust_credit->insert;
+  $cust_credit->insert(%options);
 }
 
 =item charge AMOUNT [ PKG [ COMMENT [ TAXCLASS ] ] ]
@@ -3863,16 +4900,22 @@ the error, otherwise returns false.
 
 sub charge {
   my $self = shift;
-  my ( $amount, $pkg, $comment, $taxclass, $additional );
+  my ( $amount, $quantity, $pkg, $comment, $taxclass, $additional, $classnum );
+  my ( $taxproduct, $override );
   if ( ref( $_[0] ) ) {
     $amount     = $_[0]->{amount};
+    $quantity   = exists($_[0]->{quantity}) ? $_[0]->{quantity} : 1;
     $pkg        = exists($_[0]->{pkg}) ? $_[0]->{pkg} : 'One-time charge';
     $comment    = exists($_[0]->{comment}) ? $_[0]->{comment}
                                            : '$'. sprintf("%.2f",$amount);
     $taxclass   = exists($_[0]->{taxclass}) ? $_[0]->{taxclass} : '';
+    $classnum   = exists($_[0]->{classnum}) ? $_[0]->{classnum} : '';
     $additional = $_[0]->{additional};
+    $taxproduct = $_[0]->{taxproductnum};
+    $override   = { '' => $_[0]->{tax_override} };
   }else{
     $amount     = shift;
+    $quantity   = 1;
     $pkg        = @_ ? shift : 'One-time charge';
     $comment    = @_ ? shift : '$'. sprintf("%.2f",$amount);
     $taxclass   = @_ ? shift : '';
@@ -3891,12 +4934,14 @@ sub charge {
   my $dbh = dbh;
 
   my $part_pkg = new FS::part_pkg ( {
-    'pkg'      => $pkg,
-    'comment'  => $comment,
-    'plan'     => 'flat',
-    'freq'     => 0,
-    'disabled' => 'Y',
-    'taxclass' => $taxclass,
+    'pkg'           => $pkg,
+    'comment'       => $comment,
+    'plan'          => 'flat',
+    'freq'          => 0,
+    'disabled'      => 'Y',
+    'classnum'      => $classnum ? $classnum : '',
+    'taxclass'      => $taxclass,
+    'taxproductnum' => $taxproduct,
   } );
 
   my %options = ( ( map { ("additional_info$_" => $additional->[$_] ) }
@@ -3906,7 +4951,9 @@ sub charge {
                   'setup_fee' => $amount,
                 );
 
-  my $error = $part_pkg->insert( options => \%options );
+  my $error = $part_pkg->insert( options       => \%options,
+                                 tax_overrides => $override,
+                               );
   if ( $error ) {
     $dbh->rollback if $oldAutoCommit;
     return $error;
@@ -3924,8 +4971,9 @@ sub charge {
   }
 
   my $cust_pkg = new FS::cust_pkg ( {
-    'custnum' => $self->custnum,
-    'pkgpart' => $pkgpart,
+    'custnum'  => $self->custnum,
+    'pkgpart'  => $pkgpart,
+    'quantity' => $quantity,
   } );
 
   $error = $cust_pkg->insert;
@@ -3939,6 +4987,33 @@ sub charge {
 
 }
 
+#=item charge_postal_fee
+#
+#Applies a one time charge this customer.  If there is an error,
+#returns the error, returns the cust_pkg charge object or false
+#if there was no charge.
+#
+#=cut
+#
+# This should be a customer event.  For that to work requires that bill
+# also be a customer event.
+
+sub charge_postal_fee {
+  my $self = shift;
+
+  my $pkgpart = $conf->config('postal_invoice-fee_pkgpart');
+  return '' unless ($pkgpart && grep { $_ eq 'POST' } $self->invoicing_list);
+
+  my $cust_pkg = new FS::cust_pkg ( {
+    'custnum'  => $self->custnum,
+    'pkgpart'  => $pkgpart,
+    'quantity' => 1,
+  } );
+
+  my $error = $cust_pkg->insert;
+  $error ? $error : $cust_pkg;
+}
+
 =item cust_bill
 
 Returns all the invoices (see L<FS::cust_bill>) for this customer.
@@ -3999,6 +5074,17 @@ sub cust_pay_void {
     qsearch( 'cust_pay_void', { 'custnum' => $self->custnum } )
 }
 
+=item cust_pay_batch
+
+Returns all batched payments (see L<FS::cust_pay_void>) for this customer.
+
+=cut
+
+sub cust_pay_batch {
+  my $self = shift;
+  sort { $a->_date <=> $b->_date }
+    qsearch( 'cust_pay_batch', { 'custnum' => $self->custnum } )
+}
 
 =item cust_refund
 
@@ -4079,6 +5165,40 @@ sub country_full {
   code2country($self->country);
 }
 
+=item geocode DATA_VENDOR
+
+Returns a value for the customer location as encoded by DATA_VENDOR.
+Currently this only makes sense for "CCH" as DATA_VENDOR.
+
+=cut
+
+sub geocode {
+  my ($self, $data_vendor) = (shift, shift);  #always cch for now
+
+  my $prefix = ( $conf->exists('tax-ship_address') && length($self->ship_last) )
+               ? 'ship_'
+               : '';
+
+  my ($zip,$plus4) = split /-/, $self->get("${prefix}zip")
+    if $self->country eq 'US';
+
+  #CCH specific location stuff
+  my $extra_sql = "AND plus4lo <= '$plus4' AND plus4hi >= '$plus4'";
+
+  my $geocode = '';
+  my $cust_tax_location =
+    qsearchs( {
+                'table'     => 'cust_tax_location', 
+                'hashref'   => { 'zip' => $zip, 'data_vendor' => $data_vendor },
+                'extra_sql' => $extra_sql,
+              }
+            );
+  $geocode = $cust_tax_location->geocode
+    if $cust_tax_location;
+
+  $geocode;
+}
+
 =item cust_status
 
 =item status
@@ -4113,152 +5233,627 @@ sub cust_status {
       or die "Error executing 'SELECT $sql': ". $sth->errstr;
     return $status if $sth->fetchrow_arrayref->[0];
   }
-}
+}
+
+=item ucfirst_cust_status
+
+=item ucfirst_status
+
+Returns the status with the first character capitalized.
+
+=cut
+
+sub ucfirst_status { shift->ucfirst_cust_status(@_); }
+
+sub ucfirst_cust_status {
+  my $self = shift;
+  ucfirst($self->cust_status);
+}
+
+=item statuscolor
+
+Returns a hex triplet color string for this customer's status.
+
+=cut
+
+use vars qw(%statuscolor);
+tie %statuscolor, 'Tie::IxHash',
+  'prospect'  => '7e0079', #'000000', #black?  naw, purple
+  'active'    => '00CC00', #green
+  'inactive'  => '0000CC', #blue
+  'suspended' => 'FF9900', #yellow
+  'cancelled' => 'FF0000', #red
+;
+
+sub statuscolor { shift->cust_statuscolor(@_); }
+
+sub cust_statuscolor {
+  my $self = shift;
+  $statuscolor{$self->cust_status};
+}
+
+=item tickets
+
+Returns an array of hashes representing the customer's RT tickets.
+
+=cut
+
+sub tickets {
+  my $self = shift;
+
+  my $num = $conf->config('cust_main-max_tickets') || 10;
+  my @tickets = ();
+
+  unless ( $conf->config('ticket_system-custom_priority_field') ) {
+
+    @tickets = @{ FS::TicketSystem->customer_tickets($self->custnum, $num) };
+
+  } else {
+
+    foreach my $priority (
+      $conf->config('ticket_system-custom_priority_field-values'), ''
+    ) {
+      last if scalar(@tickets) >= $num;
+      push @tickets, 
+        @{ FS::TicketSystem->customer_tickets( $self->custnum,
+                                               $num - scalar(@tickets),
+                                               $priority,
+                                             )
+         };
+    }
+  }
+  (@tickets);
+}
+
+# Return services representing svc_accts in customer support packages
+sub support_services {
+  my $self = shift;
+  my %packages = map { $_ => 1 } $conf->config('support_packages');
+
+  grep { $_->pkg_svc && $_->pkg_svc->primary_svc eq 'Y' }
+    grep { $_->part_svc->svcdb eq 'svc_acct' }
+    map { $_->cust_svc }
+    grep { exists $packages{ $_->pkgpart } }
+    $self->ncancelled_pkgs;
+
+}
+
+=back
+
+=head1 CLASS METHODS
+
+=over 4
+
+=item statuses
+
+Class method that returns the list of possible status strings for customers
+(see L<the status method|/status>).  For example:
+
+  @statuses = FS::cust_main->statuses();
+
+=cut
+
+sub statuses {
+  #my $self = shift; #could be class...
+  keys %statuscolor;
+}
+
+=item prospect_sql
+
+Returns an SQL expression identifying prospective cust_main records (customers
+with no packages ever ordered)
+
+=cut
+
+use vars qw($select_count_pkgs);
+$select_count_pkgs =
+  "SELECT COUNT(*) FROM cust_pkg
+    WHERE cust_pkg.custnum = cust_main.custnum";
+
+sub select_count_pkgs_sql {
+  $select_count_pkgs;
+}
+
+sub prospect_sql { "
+  0 = ( $select_count_pkgs )
+"; }
+
+=item active_sql
+
+Returns an SQL expression identifying active cust_main records (customers with
+active recurring packages).
+
+=cut
+
+sub active_sql { "
+  0 < ( $select_count_pkgs AND ". FS::cust_pkg->active_sql. "
+      )
+"; }
+
+=item inactive_sql
+
+Returns an SQL expression identifying inactive cust_main records (customers with
+no active recurring packages, but otherwise unsuspended/uncancelled).
+
+=cut
+
+sub inactive_sql { "
+  0 = ( $select_count_pkgs AND ". FS::cust_pkg->active_sql. " )
+  AND
+  0 < ( $select_count_pkgs AND ". FS::cust_pkg->inactive_sql. " )
+"; }
+
+=item susp_sql
+=item suspended_sql
+
+Returns an SQL expression identifying suspended cust_main records.
+
+=cut
+
+
+sub suspended_sql { susp_sql(@_); }
+sub susp_sql { "
+    0 < ( $select_count_pkgs AND ". FS::cust_pkg->suspended_sql. " )
+    AND
+    0 = ( $select_count_pkgs AND ". FS::cust_pkg->active_sql. " )
+"; }
+
+=item cancel_sql
+=item cancelled_sql
+
+Returns an SQL expression identifying cancelled cust_main records.
+
+=cut
+
+sub cancelled_sql { cancel_sql(@_); }
+sub cancel_sql {
+
+  my $recurring_sql = FS::cust_pkg->recurring_sql;
+  my $cancelled_sql = FS::cust_pkg->cancelled_sql;
+
+  "
+        0 < ( $select_count_pkgs )
+    AND 0 < ( $select_count_pkgs AND $recurring_sql AND $cancelled_sql   )
+    AND 0 = ( $select_count_pkgs AND $recurring_sql
+                  AND ( cust_pkg.cancel IS NULL OR cust_pkg.cancel = 0 )
+            )
+    AND 0 = (  $select_count_pkgs AND ". FS::cust_pkg->inactive_sql. " )
+  ";
+
+}
+
+=item uncancel_sql
+=item uncancelled_sql
+
+Returns an SQL expression identifying un-cancelled cust_main records.
+
+=cut
+
+sub uncancelled_sql { uncancel_sql(@_); }
+sub uncancel_sql { "
+  ( 0 < ( $select_count_pkgs
+                   AND ( cust_pkg.cancel IS NULL
+                         OR cust_pkg.cancel = 0
+                       )
+        )
+    OR 0 = ( $select_count_pkgs )
+  )
+"; }
+
+=item balance_sql
+
+Returns an SQL fragment to retreive the balance.
+
+=cut
+
+sub balance_sql { "
+    ( SELECT COALESCE( SUM(charged), 0 ) FROM cust_bill
+        WHERE cust_bill.custnum   = cust_main.custnum     )
+  - ( SELECT COALESCE( SUM(paid),    0 ) FROM cust_pay
+        WHERE cust_pay.custnum    = cust_main.custnum     )
+  - ( SELECT COALESCE( SUM(amount),  0 ) FROM cust_credit
+        WHERE cust_credit.custnum = cust_main.custnum     )
+  + ( SELECT COALESCE( SUM(refund),  0 ) FROM cust_refund
+        WHERE cust_refund.custnum = cust_main.custnum     )
+"; }
+
+=item balance_date_sql START_TIME [ END_TIME [ OPTION => VALUE ... ] ]
+
+Returns an SQL fragment to retreive the balance for this customer, only
+considering invoices with date earlier than START_TIME, and optionally not
+later than END_TIME (total_owed_date minus total_credited minus
+total_unapplied_payments).
+
+Times are specified as SQL fragments or numeric
+UNIX timestamps; see L<perlfunc/"time">).  Also see L<Time::Local> and
+L<Date::Parse> for conversion functions.  The empty string can be passed
+to disable that time constraint completely.
+
+Available options are:
+
+=over 4
+
+=item unapplied_date
+
+set to true to disregard unapplied credits, payments and refunds outside the specified time period - by default the time period restriction only applies to invoices (useful for reporting, probably a bad idea for event triggering)
+
+=item total
+
+(unused.  obsolete?)
+set to true to remove all customer comparison clauses, for totals
+
+=item where
+
+(unused.  obsolete?)
+WHERE clause hashref (elements "AND"ed together) (typically used with the total option)
+
+=item join
+
+(unused.  obsolete?)
+JOIN clause (typically used with the total option)
+
+=back
+
+=cut
+
+sub balance_date_sql {
+  my( $class, $start, $end, %opt ) = @_;
+
+  my $owed         = FS::cust_bill->owed_sql;
+  my $unapp_refund = FS::cust_refund->unapplied_sql;
+  my $unapp_credit = FS::cust_credit->unapplied_sql;
+  my $unapp_pay    = FS::cust_pay->unapplied_sql;
+
+  my $j = $opt{'join'} || '';
+
+  my $owed_wh   = $class->_money_table_where( 'cust_bill',   $start,$end,%opt );
+  my $refund_wh = $class->_money_table_where( 'cust_refund', $start,$end,%opt );
+  my $credit_wh = $class->_money_table_where( 'cust_credit', $start,$end,%opt );
+  my $pay_wh    = $class->_money_table_where( 'cust_pay',    $start,$end,%opt );
+
+  "   ( SELECT COALESCE(SUM($owed),         0) FROM cust_bill   $j $owed_wh   )
+    + ( SELECT COALESCE(SUM($unapp_refund), 0) FROM cust_refund $j $refund_wh )
+    - ( SELECT COALESCE(SUM($unapp_credit), 0) FROM cust_credit $j $credit_wh )
+    - ( SELECT COALESCE(SUM($unapp_pay),    0) FROM cust_pay    $j $pay_wh    )
+  ";
+
+}
+
+=item _money_table_where TABLE START_TIME [ END_TIME [ OPTION => VALUE ... ] ]
+
+Helper method for balance_date_sql; name (and usage) subject to change
+(suggestions welcome).
+
+Returns a WHERE clause for the specified monetary TABLE (cust_bill,
+cust_refund, cust_credit or cust_pay).
+
+If TABLE is "cust_bill" or the unapplied_date option is true, only
+considers records with date earlier than START_TIME, and optionally not
+later than END_TIME .
+
+=cut
+
+sub _money_table_where {
+  my( $class, $table, $start, $end, %opt ) = @_;
+
+  my @where = ();
+  push @where, "cust_main.custnum = $table.custnum" unless $opt{'total'};
+  if ( $table eq 'cust_bill' || $opt{'unapplied_date'} ) {
+    push @where, "$table._date <= $start" if defined($start) && length($start);
+    push @where, "$table._date >  $end"   if defined($end)   && length($end);
+  }
+  push @where, @{$opt{'where'}} if $opt{'where'};
+  my $where = scalar(@where) ? 'WHERE '. join(' AND ', @where ) : '';
+
+  $where;
+
+}
+
+=item search_sql HASHREF
+
+(Class method)
+
+Returns a qsearch hash expression to search for parameters specified in HREF.
+Valid parameters are
+
+=over 4
+
+=item agentnum
+
+=item status
+
+=item cancelled_pkgs
+
+bool
+
+=item signupdate
+
+listref of start date, end date
+
+=item payby
+
+listref
+
+=item current_balance
+
+listref (list returned by FS::UI::Web::parse_lt_gt($cgi, 'current_balance'))
+
+=item cust_fields
+
+=item flattened_pkgs
+
+bool
+
+=back
+
+=cut
+
+sub search_sql {
+  my ($class, $params) = @_;
+
+  my $dbh = dbh;
+
+  my @where = ();
+  my $orderby;
+
+  ##
+  # parse agent
+  ##
+
+  if ( $params->{'agentnum'} =~ /^(\d+)$/ and $1 ) {
+    push @where,
+      "cust_main.agentnum = $1";
+  }
+
+  ##
+  # parse status
+  ##
+
+  #prospect active inactive suspended cancelled
+  if ( grep { $params->{'status'} eq $_ } FS::cust_main->statuses() ) {
+    my $method = $params->{'status'}. '_sql';
+    #push @where, $class->$method();
+    push @where, FS::cust_main->$method();
+  }
+  
+  ##
+  # parse cancelled package checkbox
+  ##
+
+  my $pkgwhere = "";
+
+  $pkgwhere .= "AND (cancel = 0 or cancel is null)"
+    unless $params->{'cancelled_pkgs'};
+
+  ##
+  # dates
+  ##
+
+  foreach my $field (qw( signupdate )) {
+
+    next unless exists($params->{$field});
+
+    my($beginning, $ending) = @{$params->{$field}};
+
+    push @where,
+      "cust_main.$field IS NOT NULL",
+      "cust_main.$field >= $beginning",
+      "cust_main.$field <= $ending";
+
+    $orderby ||= "ORDER BY cust_main.$field";
+
+  }
+
+  ###
+  # payby
+  ###
+
+  my @payby = grep /^([A-Z]{4})$/, @{ $params->{'payby'} };
+  if ( @payby ) {
+    push @where, '( '. join(' OR ', map "cust_main.payby = '$_'", @payby). ' )';
+  }
 
-=item ucfirst_cust_status
+  ##
+  # amounts
+  ##
 
-=item ucfirst_status
+  #my $balance_sql = $class->balance_sql();
+  my $balance_sql = FS::cust_main->balance_sql();
 
-Returns the status with the first character capitalized.
+  push @where, map { s/current_balance/$balance_sql/; $_ }
+                   @{ $params->{'current_balance'} };
 
-=cut
+  ##
+  # custbatch
+  ##
 
-sub ucfirst_status { shift->ucfirst_cust_status(@_); }
+  if ( $params->{'custbatch'} =~ /^([\w\/\-\:\.]+)$/ and $1 ) {
+    push @where,
+      "cust_main.custbatch = '$1'";
+  }
 
-sub ucfirst_cust_status {
-  my $self = shift;
-  ucfirst($self->cust_status);
-}
+  ##
+  # setup queries, subs, etc. for the search
+  ##
 
-=item statuscolor
+  $orderby ||= 'ORDER BY custnum';
 
-Returns a hex triplet color string for this customer's status.
+  # here is the agent virtualization
+  push @where, $FS::CurrentUser::CurrentUser->agentnums_sql;
 
-=cut
+  my $extra_sql = scalar(@where) ? ' WHERE '. join(' AND ', @where) : '';
 
-use vars qw(%statuscolor);
-%statuscolor = (
-  'prospect'  => '7e0079', #'000000', #black?  naw, purple
-  'active'    => '00CC00', #green
-  'inactive'  => '0000CC', #blue
-  'suspended' => 'FF9900', #yellow
-  'cancelled' => 'FF0000', #red
-);
+  my $addl_from = 'LEFT JOIN cust_pkg USING ( custnum  ) ';
 
-sub statuscolor { shift->cust_statuscolor(@_); }
+  my $count_query = "SELECT COUNT(*) FROM cust_main $extra_sql";
+
+  my $select = join(', ', 
+                 'cust_main.custnum',
+                 FS::UI::Web::cust_sql_fields($params->{'cust_fields'}),
+               );
+
+  my(@extra_headers) = ();
+  my(@extra_fields)  = ();
+
+  if ($params->{'flattened_pkgs'}) {
+
+    if ($dbh->{Driver}->{Name} eq 'Pg') {
+
+      $select .= ", array_to_string(array(select pkg from cust_pkg left join part_pkg using ( pkgpart ) where cust_main.custnum = cust_pkg.custnum $pkgwhere),'|') as magic";
+
+    }elsif ($dbh->{Driver}->{Name} =~ /^mysql/i) {
+      $select .= ", GROUP_CONCAT(pkg SEPARATOR '|') as magic";
+      $addl_from .= " LEFT JOIN part_pkg using ( pkgpart )";
+    }else{
+      warn "warning: unknown database type ". $dbh->{Driver}->{Name}. 
+           "omitting packing information from report.";
+    }
+
+    my $header_query = "SELECT COUNT(cust_pkg.custnum = cust_main.custnum) AS count FROM cust_main $addl_from $extra_sql $pkgwhere group by cust_main.custnum order by count desc limit 1";
+
+    my $sth = dbh->prepare($header_query) or die dbh->errstr;
+    $sth->execute() or die $sth->errstr;
+    my $headerrow = $sth->fetchrow_arrayref;
+    my $headercount = $headerrow ? $headerrow->[0] : 0;
+    while($headercount) {
+      unshift @extra_headers, "Package ". $headercount;
+      unshift @extra_fields, eval q!sub {my $c = shift;
+                                         my @a = split '\|', $c->magic;
+                                         my $p = $a[!.--$headercount. q!];
+                                         $p;
+                                        };!;
+    }
+
+  }
+
+  my $sql_query = {
+    'table'         => 'cust_main',
+    'select'        => $select,
+    'hashref'       => {},
+    'extra_sql'     => $extra_sql,
+    'order_by'      => $orderby,
+    'count_query'   => $count_query,
+    'extra_headers' => \@extra_headers,
+    'extra_fields'  => \@extra_fields,
+  };
 
-sub cust_statuscolor {
-  my $self = shift;
-  $statuscolor{$self->cust_status};
 }
 
-=back
+=item email_search_sql HASHREF
 
-=head1 CLASS METHODS
+(Class method)
+
+Emails a notice to the specified customers.
+
+Valid parameters are those of the L<search_sql> method, plus the following:
 
 =over 4
 
-=item prospect_sql
+=item from
 
-Returns an SQL expression identifying prospective cust_main records (customers
-with no packages ever ordered)
+From: address
 
-=cut
+=item subject
 
-use vars qw($select_count_pkgs);
-$select_count_pkgs =
-  "SELECT COUNT(*) FROM cust_pkg
-    WHERE cust_pkg.custnum = cust_main.custnum";
+Email Subject:
 
-sub select_count_pkgs_sql {
-  $select_count_pkgs;
-}
+=item html_body
 
-sub prospect_sql { "
-  0 = ( $select_count_pkgs )
-"; }
+HTML body
 
-=item active_sql
+=item text_body
 
-Returns an SQL expression identifying active cust_main records (customers with
-no active recurring packages, but otherwise unsuspended/uncancelled).
+Text body
 
-=cut
+=item job
 
-sub active_sql { "
-  0 < ( $select_count_pkgs AND ". FS::cust_pkg->active_sql. "
-      )
-"; }
+Optional job queue job for status updates.
 
-=item inactive_sql
+=back
 
-Returns an SQL expression identifying inactive cust_main records (customers with
-active recurring packages).
+Returns an error message, or false for success.
+
+If an error occurs during any email, stops the enture send and returns that
+error.  Presumably if you're getting SMTP errors aborting is better than 
+retrying everything.
 
 =cut
 
-sub inactive_sql { "
-  0 = ( $select_count_pkgs AND ". FS::cust_pkg->active_sql. " )
-  AND
-  0 < ( $select_count_pkgs AND ". FS::cust_pkg->inactive_sql. " )
-"; }
+sub email_search_sql {
+  my($class, $params) = @_;
 
-=item susp_sql
-=item suspended_sql
+  my $from = delete $params->{from};
+  my $subject = delete $params->{subject};
+  my $html_body = delete $params->{html_body};
+  my $text_body = delete $params->{text_body};
 
-Returns an SQL expression identifying suspended cust_main records.
+  my $job = delete $params->{'job'};
 
-=cut
+  my $sql_query = $class->search_sql($params);
 
+  my $count_query   = delete($sql_query->{'count_query'});
+  my $count_sth = dbh->prepare($count_query)
+    or die "Error preparing $count_query: ". dbh->errstr;
+  $count_sth->execute
+    or die "Error executing $count_query: ". $count_sth->errstr;
+  my $count_arrayref = $count_sth->fetchrow_arrayref;
+  my $num_cust = $count_arrayref->[0];
 
-sub suspended_sql { susp_sql(@_); }
-sub susp_sql { "
-    0 < ( $select_count_pkgs AND ". FS::cust_pkg->suspended_sql. " )
-    AND
-    0 = ( $select_count_pkgs AND ". FS::cust_pkg->active_sql. " )
-"; }
+  #my @extra_headers = @{ delete($sql_query->{'extra_headers'}) };
+  #my @extra_fields  = @{ delete($sql_query->{'extra_fields'})  };
 
-=item cancel_sql
-=item cancelled_sql
 
-Returns an SQL expression identifying cancelled cust_main records.
+  my( $num, $last, $min_sec ) = (0, time, 5); #progresbar foo
 
-=cut
+  #eventually order+limit magic to reduce memory use?
+  foreach my $cust_main ( qsearch($sql_query) ) {
 
-sub cancelled_sql { cancel_sql(@_); }
-sub cancel_sql {
+    my $to = $cust_main->invoicing_list_emailonly_scalar;
+    next unless $to;
 
-  my $recurring_sql = FS::cust_pkg->recurring_sql;
-  #my $recurring_sql = "
-  #  '0' != ( select freq from part_pkg
-  #             where cust_pkg.pkgpart = part_pkg.pkgpart )
-  #";
+    my $error = send_email(
+      generate_email(
+        'from'      => $from,
+        'to'        => $to,
+        'subject'   => $subject,
+        'html_body' => $html_body,
+        'text_body' => $text_body,
+      )
+    );
+    return $error if $error;
 
-  "
-    0 < ( $select_count_pkgs )
-    AND 0 = ( $select_count_pkgs AND $recurring_sql
-                  AND ( cust_pkg.cancel IS NULL OR cust_pkg.cancel = 0 )
-            )
-  ";
+    if ( $job ) { #progressbar foo
+      $num++;
+      if ( time - $min_sec > $last ) {
+        my $error = $job->update_statustext(
+          int( 100 * $num / $num_cust )
+        );
+        die $error if $error;
+        $last = time;
+      }
+    }
+
+  }
+
+  return '';
 }
 
-=item uncancel_sql
-=item uncancelled_sql
+use Storable qw(thaw);
+use Data::Dumper;
+use MIME::Base64;
+sub process_email_search_sql {
+  my $job = shift;
+  #warn "$me process_re_X $method for job $job\n" if $DEBUG;
 
-Returns an SQL expression identifying un-cancelled cust_main records.
+  my $param = thaw(decode_base64(shift));
+  warn Dumper($param) if $DEBUG;
 
-=cut
+  $param->{'job'} = $job;
 
-sub uncancelled_sql { uncancel_sql(@_); }
-sub uncancel_sql { "
-  ( 0 < ( $select_count_pkgs
-                   AND ( cust_pkg.cancel IS NULL
-                         OR cust_pkg.cancel = 0
-                       )
-        )
-    OR 0 = ( $select_count_pkgs )
-  )
-"; }
+  my $error = FS::cust_main->email_search_sql( $param );
+  die $error if $error;
+
+}
 
 =item fuzzy_search FUZZY_HASHREF [ HASHREF, SELECT, EXTRA_SQL, CACHE_OBJ ]
 
@@ -4388,7 +5983,14 @@ sub smart_search {
 
     }
 
-  } elsif ( $search =~ /^\s*(\d+)\s*$/ ) { # customer # search
+  # custnum search (also try agent_custid), with some tweaking options if your
+  # legacy cust "numbers" have letters
+  } elsif ( $search =~ /^\s*(\d+)\s*$/
+            || ( $conf->config('cust_main-agent_custid-format') eq 'ww?d+'
+                 && $search =~ /^\s*(\w\w?\d+)\s*$/
+               )
+          )
+  {
 
     push @cust_main, qsearch( {
       'table'     => 'cust_main',
@@ -4396,6 +5998,12 @@ sub smart_search {
       'extra_sql' => " AND $agentnums_sql", #agent virtualization
     } );
 
+    push @cust_main, qsearch( {
+      'table'     => 'cust_main',
+      'hashref'   => { 'agent_custid' => $1, %options },
+      'extra_sql' => " AND $agentnums_sql", #agent virtualization
+    } );
+
   } elsif ( $search =~ /^\s*(\S.*\S)\s+\((.+), ([^,]+)\)\s*$/ ) {
 
     my($company, $last, $first) = ( $1, $2, $3 );
@@ -4483,11 +6091,12 @@ sub smart_search {
       'extra_sql' => "$sql AND $agentnums_sql", #agent virtualization
     } );
 
-    #always do substring & fuzzy,
-    #getting complains searches are not returning enough
-    unless ( @cust_main && $skip_fuzzy ) {  #no exact match, trying substring/fuzzy
+    #no exact match, trying substring/fuzzy
+    #always do substring & fuzzy (unless they're explicity config'ed off)
+    #getting complaints searches are not returning enough
+    unless ( @cust_main  && $skip_fuzzy || $conf->exists('disable-fuzzy') ) {
 
-      #still some false laziness w/ search/cust_main.cgi
+      #still some false laziness w/search_sql (was search/cust_main.cgi)
 
       #substring
 
@@ -4558,6 +6167,72 @@ sub smart_search {
 
 }
 
+=item email_search
+
+Accepts the following options: I<email>, the email address to search for.  The
+email address will be searched for as an email invoice destination and as an
+svc_acct account.
+
+#Any additional options are treated as an additional qualifier on the search
+#(i.e. I<agentnum>).
+
+Returns a (possibly empty) array of FS::cust_main objects (but usually just
+none or one).
+
+=cut
+
+sub email_search {
+  my %options = @_;
+
+  local($DEBUG) = 1;
+
+  my $email = delete $options{'email'};
+
+  #we're only being used by RT at the moment... no agent virtualization yet
+  #my $agentnums_sql = $FS::CurrentUser::CurrentUser->agentnums_sql;
+
+  my @cust_main = ();
+
+  if ( $email =~ /([^@]+)\@([^@]+)/ ) {
+
+    my ( $user, $domain ) = ( $1, $2 );
+
+    warn "$me smart_search: searching for $user in domain $domain"
+      if $DEBUG;
+
+    push @cust_main,
+      map $_->cust_main,
+          qsearch( {
+                     'table'     => 'cust_main_invoice',
+                     'hashref'   => { 'dest' => $email },
+                   }
+                 );
+
+    push @cust_main,
+      map  $_->cust_main,
+      grep $_,
+      map  $_->cust_svc->cust_pkg,
+          qsearch( {
+                     'table'     => 'svc_acct',
+                     'hashref'   => { 'username' => $user, },
+                     'extra_sql' =>
+                       'AND ( SELECT domain FROM svc_domain
+                                WHERE svc_acct.domsvc = svc_domain.svcnum
+                            ) = '. dbh->quote($domain),
+                   }
+                 );
+  }
+
+  my %saw = ();
+  @cust_main = grep { !$saw{$_->custnum}++ } @cust_main;
+
+  warn "$me smart_search: found ". scalar(@cust_main). " unique customers"
+    if $DEBUG;
+
+  @cust_main;
+
+}
+
 =item check_and_rebuild_fuzzyfiles
 
 =cut
@@ -4659,21 +6334,79 @@ sub append_fuzzyfiles {
   1;
 }
 
+=item process_batch_import
+
+Load a batch import as a queued JSRPC job
+
+=cut
+
+use Storable qw(thaw);
+use Data::Dumper;
+use MIME::Base64;
+sub process_batch_import {
+  my $job = shift;
+
+  my $param = thaw(decode_base64(shift));
+  warn Dumper($param) if $DEBUG;
+  
+  my $files = $param->{'uploaded_files'}
+    or die "No files provided.\n";
+
+  my (%files) = map { /^(\w+):([\.\w]+)$/ ? ($1,$2):() } split /,/, $files;
+
+  my $dir = '%%%FREESIDE_CACHE%%%/cache.'. $FS::UID::datasrc. '/';
+  my $file = $dir. $files{'file'};
+
+  my $type;
+  if ( $file =~ /\.(\w+)$/i ) {
+    $type = lc($1);
+  } else {
+    #or error out???
+    warn "can't parse file type from filename $file; defaulting to CSV";
+    $type = 'csv';
+  }
+
+  my $error =
+    FS::cust_main::batch_import( {
+      job       => $job,
+      file      => $file,
+      type      => $type,
+      custbatch => $param->{custbatch},
+      agentnum  => $param->{'agentnum'},
+      refnum    => $param->{'refnum'},
+      pkgpart   => $param->{'pkgpart'},
+      #'fields'  => [qw( cust_pkg.setup dayphone first last address1 address2
+      #                 city state zip comments                          )],
+      'format'  => $param->{'format'},
+    } );
+
+  unlink $file;
+
+  die "$error\n" if $error;
+
+}
+
 =item batch_import
 
 =cut
 
+#some false laziness w/cdr.pm now
 sub batch_import {
   my $param = shift;
-  #warn join('-',keys %$param);
-  my $fh = $param->{filehandle};
-  my $agentnum = $param->{agentnum};
 
-  my $refnum = $param->{refnum};
-  my $pkgpart = $param->{pkgpart};
+  my $job       = $param->{job};
+
+  my $filename  = $param->{file};
+  my $type      = $param->{type} || 'csv';
+
+  my $custbatch = $param->{custbatch};
+
+  my $agentnum  = $param->{agentnum};
+  my $refnum    = $param->{refnum};
+  my $pkgpart   = $param->{pkgpart};
+
+  my $format    = $param->{'format'};
 
-  #my @fields = @{$param->{fields}};
-  my $format = $param->{'format'};
   my @fields;
   my $payby;
   if ( $format eq 'simple' ) {
@@ -4692,18 +6425,50 @@ sub batch_import {
                   svc_acct.username svc_acct._password 
                 );
     $payby = 'BILL';
+ } elsif ( $format eq 'extended-plus_company' ) {
+    @fields = qw( agent_custid refnum
+                  last first company address1 address2 city state zip country
+                  daytime night
+                  ship_last ship_first ship_company ship_address1 ship_address2
+                  ship_city ship_state ship_zip ship_country
+                  payinfo paycvv paydate
+                  invoicing_list
+                  cust_pkg.pkgpart
+                  svc_acct.username svc_acct._password 
+                );
+    $payby = 'BILL';
   } else {
     die "unknown format $format";
   }
 
-  eval "use Text::CSV_XS;";
-  die $@ if $@;
+  my $count;
+  my $parser;
+  my @buffer = ();
+  if ( $type eq 'csv' ) {
 
-  my $csv = new Text::CSV_XS;
-  #warn $csv;
-  #warn $fh;
+    eval "use Text::CSV_XS;";
+    die $@ if $@;
+
+    $parser = new Text::CSV_XS;
+
+    @buffer = split(/\r?\n/, slurp($filename) );
+    $count = scalar(@buffer);
+
+  } elsif ( $type eq 'xls' ) {
+
+    eval "use Spreadsheet::ParseExcel;";
+    die $@ if $@;
+
+    my $excel = new Spreadsheet::ParseExcel::Workbook->Parse($filename);
+    $parser = $excel->{Worksheet}[0]; #first sheet
+
+    $count = $parser->{MaxRow} || $parser->{MinRow};
+    $count++;
+
+  } else {
+    die "Unknown file type $type\n";
+  }
 
-  my $imported = 0;
   #my $columns;
 
   local $SIG{HUP} = 'IGNORE';
@@ -4717,24 +6482,46 @@ sub batch_import {
   local $FS::UID::AutoCommit = 0;
   my $dbh = dbh;
   
-  #while ( $columns = $csv->getline($fh) ) {
   my $line;
-  while ( defined($line=<$fh>) ) {
+  my $row = 0;
+  my( $last, $min_sec ) = ( time, 5 ); #progressbar foo
+  while (1) {
 
-    $csv->parse($line) or do {
-      $dbh->rollback if $oldAutoCommit;
-      return "can't parse: ". $csv->error_input();
-    };
+    my @columns = ();
+    if ( $type eq 'csv' ) {
+
+      last unless scalar(@buffer);
+      $line = shift(@buffer);
+
+      $parser->parse($line) or do {
+        $dbh->rollback if $oldAutoCommit;
+        return "can't parse: ". $parser->error_input();
+      };
+      @columns = $parser->fields();
+
+    } elsif ( $type eq 'xls' ) {
+
+      last if $row > ($parser->{MaxRow} || $parser->{MinRow});
+
+      my @row = @{ $parser->{Cells}[$row] };
+      @columns = map $_->{Val}, @row;
+
+      #my $z = 'A';
+      #warn $z++. ": $_\n" for @columns;
+
+    } else {
+      die "Unknown file type $type\n";
+    }
 
-    my @columns = $csv->fields();
     #warn join('-',@columns);
 
     my %cust_main = (
-      agentnum => $agentnum,
-      refnum   => $refnum,
-      country  => $conf->config('countrydefault') || 'US',
-      payby    => $payby, #default
-      paydate  => '12/2037', #default
+      custbatch => $custbatch,
+      agentnum  => $agentnum,
+      refnum    => $refnum,
+      country   => $conf->config('countrydefault') || 'US',
+      payby     => $payby, #default
+      paydate   => '12/2037', #default
     );
     my $billtime = time;
     my %cust_pkg = ( pkgpart => $pkgpart );
@@ -4781,12 +6568,14 @@ sub batch_import {
           $columns[0] = $part_referral->refnum;
         }
 
-        #$cust_main{$field} = shift @$columns; 
-        $cust_main{$field} = shift @columns; 
+        my $value = shift @columns;
+        $cust_main{$field} = $value if length($value);
       }
     }
 
-    $cust_main{'payby'} = 'CARD' if length($cust_main{'payinfo'});
+    $cust_main{'payby'} = 'CARD'
+      if defined $cust_main{'payinfo'}
+      && length  $cust_main{'payinfo'};
 
     my $invoicing_list = $cust_main{'invoicing_list'}
                            ? [ delete $cust_main{'invoicing_list'} ]
@@ -4805,7 +6594,7 @@ sub batch_import {
         my $part_pkg = $cust_pkg->part_pkg;
 	unless ( $part_pkg ) {
 	  $dbh->rollback if $oldAutoCommit;
-	  return "unknown pkgnum ". $cust_pkg{'pkgpart'};
+	  return "unknown pkgpart: ". $cust_pkg{'pkgpart'};
 	} 
         $svc_acct{svcpart} = $part_pkg->svcpart( 'svc_acct' );
         push @svc_acct, new FS::svc_acct ( \%svc_acct )
@@ -4818,7 +6607,7 @@ sub batch_import {
 
     if ( $error ) {
       $dbh->rollback if $oldAutoCommit;
-      return "can't insert customer for $line: $error";
+      return "can't insert customer". ( $line ? " for $line" : '' ). ": $error";
     }
 
     if ( $format eq 'simple' ) {
@@ -4830,8 +6619,12 @@ sub batch_import {
         return "can't bill customer for $line: $error";
       }
   
-      $cust_main->apply_payments_and_credits;
-  
+      $error = $cust_main->apply_payments_and_credits;
+      if ( $error ) {
+        $dbh->rollback if $oldAutoCommit;
+        return "can't bill customer for $line: $error";
+      }
+
       $error = $cust_main->collect();
       if ( $error ) {
         $dbh->rollback if $oldAutoCommit;
@@ -4840,12 +6633,18 @@ sub batch_import {
 
     }
 
-    $imported++;
+    $row++;
+
+    if ( $job && time - $min_sec > $last ) { #progress bar
+      $job->update_statustext( int(100 * $row / $count) );
+      $last = time;
+    }
+
   }
 
-  $dbh->commit or die $dbh->errstr if $oldAutoCommit;
+  $dbh->commit or die $dbh->errstr if $oldAutoCommit;;
 
-  return "Empty file!" unless $imported;
+  return "Empty file!" unless $row;
 
   ''; #no error
 
@@ -4986,7 +6785,11 @@ sub notify {
   $notify_template->compile()
     or die "can't compile template: Text::Template::ERROR";
 
-  my $paydate = $customer->paydate;
+  $FS::notify_template::_template::company_name = $conf->config('company_name');
+  $FS::notify_template::_template::company_address =
+    join("\n", $conf->config('company_address') ). "\n";
+
+  my $paydate = $customer->paydate || '2037-12-31';
   $FS::notify_template::_template::first = $customer->first;
   $FS::notify_template::_template::last = $customer->last;
   $FS::notify_template::_template::company = $customer->company;
@@ -5039,7 +6842,7 @@ I<$payby> - a description of the method of payment for the customer
             # would be nice to use FS::payby::shortname
 I<$payinfo> - the masked account information used to collect for this customer
 I<$expdate> - the expiration of the customer payment method in seconds from epoch
-I<$returnaddress> - the return address defaults to invoice_latexreturnaddress
+I<$returnaddress> - the return address defaults to invoice_latexreturnaddress or company_address
 
 =cut
 
@@ -5061,7 +6864,9 @@ sub generate_letter {
   my %letter_data = map { $_ => $self->$_ } $self->fields;
   $letter_data{payinfo} = $self->mask_payinfo;
 
-  my $paydate = $self->paydate;
+  #my $paydate = $self->paydate || '2037-12-31';
+  my $paydate = $self->paydate =~ /^\S+$/ ? $self->paydate : '2037-12-31';
+
   my $payby = $self->payby;
   my ($payyear,$paymonth,$payday) = split (/-/,$paydate);
   my $expire_time = timelocal(0,0,0,$payday,--$paymonth,$payyear);
@@ -5085,14 +6890,24 @@ sub generate_letter {
 
   unless(exists($letter_data{returnaddress})){
     my $retadd = join("\n", $conf->config_orbase( 'invoice_latexreturnaddress',
-                                                  $self->_agent_template)
+                                                  $self->agent_template)
                      );
-
-    $letter_data{returnaddress} = length($retadd) ? $retadd : '~';
+    if ( length($retadd) ) {
+      $letter_data{returnaddress} = $retadd;
+    } elsif ( grep /\S/, $conf->config('company_address') ) {
+      $letter_data{returnaddress} =
+        join( '\\*'."\n", map s/( {2,})/'~' x length($1)/eg,
+                          $conf->config('company_address')
+        );
+    } else {
+      $letter_data{returnaddress} = '~';
+    }
   }
 
   $letter_data{conf_dir} = "$FS::UID::conf_dir/conf.$FS::UID::datasrc";
 
+  $letter_data{company_name} = $conf->config('company_name');
+
   my $dir = $FS::UID::conf_dir."cache.". $FS::UID::datasrc;
   my $fh = new File::Temp( TEMPLATE => 'letter.'. $self->custnum. '.XXXXXXXX',
                            DIR      => $dir,
@@ -5154,32 +6969,73 @@ sub agent_invoice_from {
 sub _agent_plandata {
   my( $self, $option ) = @_;
 
-  my $part_bill_event = qsearchs( 'part_bill_event',
-    {
-      'payby'     => $self->payby,
-      'plan'      => 'send_agent',
-      'plandata'  => { 'op'    => '~',
-                       'value' => "(^|\n)agentnum ".
-                                   '([0-9]*, )*'.
-                                  $self->agentnum.
-                                   '(, [0-9]*)*'.
-                                  "(\n|\$)",
-                     },
-    },
-    '',
-    'ORDER BY seconds LIMIT 1'
-  );
+  #yuck.  this whole thing needs to be reconciled better with 1.9's idea of
+  #agent-specific Conf
 
-  return '' unless $part_bill_event;
+  use FS::part_event::Condition;
+  
+  my $agentnum = $self->agentnum;
 
-  if ( $part_bill_event->plandata =~ /^$option (.*)$/m ) {
-    return $1;
+  my $regexp = '';
+  if ( driver_name =~ /^Pg/i ) {
+    $regexp = '~';
+  } elsif ( driver_name =~ /^mysql/i ) {
+    $regexp = 'REGEXP';
   } else {
-    warn "can't parse part_bill_event eventpart#". $part_bill_event->eventpart.
-         " plandata for $option";
+    die "don't know how to use regular expressions in ". driver_name. " databases";
+  }
+
+  my $part_event_option =
+    qsearchs({
+      'select'    => 'part_event_option.*',
+      'table'     => 'part_event_option',
+      'addl_from' => q{
+        LEFT JOIN part_event USING ( eventpart )
+        LEFT JOIN part_event_option AS peo_agentnum
+          ON ( part_event.eventpart = peo_agentnum.eventpart
+               AND peo_agentnum.optionname = 'agentnum'
+               AND peo_agentnum.optionvalue }. $regexp. q{ '(^|,)}. $agentnum. q{(,|$)'
+             )
+        LEFT JOIN part_event_option AS peo_cust_bill_age
+          ON ( part_event.eventpart = peo_cust_bill_age.eventpart
+               AND peo_cust_bill_age.optionname = 'cust_bill_age'
+             )
+      },
+      #'hashref'   => { 'optionname' => $option },
+      #'hashref'   => { 'part_event_option.optionname' => $option },
+      'extra_sql' =>
+        " WHERE part_event_option.optionname = ". dbh->quote($option).
+        " AND action = 'cust_bill_send_agent' ".
+        " AND ( disabled IS NULL OR disabled != 'Y' ) ".
+        " AND peo_agentnum.optionname = 'agentnum' ".
+        " AND agentnum IS NULL OR agentnum = $agentnum ".
+        " ORDER BY
+           CASE WHEN peo_cust_bill_age.optionname != 'cust_bill_age'
+           THEN -1
+	   ELSE ". FS::part_event::Condition->age2seconds_sql('peo_cust_bill_age.optionvalue').
+        " END
+          , part_event.weight".
+        " LIMIT 1"
+    });
+    
+  unless ( $part_event_option ) {
+    return $self->agent->invoice_template || ''
+      if $option eq 'agent_templatename';
     return '';
   }
 
+  $part_event_option->optionvalue;
+
+}
+
+sub queued_bill {
+  ## actual sub, not a method, designed to be called from the queue.
+  ## sets up the customer, and calls the bill_and_collect
+  my (%args) = @_; #, ($time, $invoice_time, $check_freq, $resetup) = @_;
+  my $cust_main = qsearchs( 'cust_main', { custnum => $args{'custnum'} } );
+      $cust_main->bill_and_collect(
+        %args,
+      );
 }
 
 =back
@@ -5206,6 +7062,8 @@ Birthdates rely on negative epoch values.
 The payby for card/check batches is broken.  With mixed batching, bad
 things will happen.
 
+B<collect> I<invoice_time> should be renamed I<time>, like B<bill>.
+
 =head1 SEE ALSO
 
 L<FS::Record>, L<FS::cust_pkg>, L<FS::cust_bill>, L<FS::cust_credit>