X-Git-Url: http://git.freeside.biz/gitweb/?p=freeside.git;a=blobdiff_plain;f=FS%2FFS%2Fcust_main%2FBilling_Realtime.pm;h=b65860e9b1ec4b207885721f8a078706ed53dad2;hp=3757ca814ddd4946bd124600b78fdd3e692ec268;hb=4fd1280540e2c9b90fa59c0c32d691f5222f65d4;hpb=3676ccddeab727d3d7a929b1fe4fe19d81e0e8c0

diff --git a/FS/FS/cust_main/Billing_Realtime.pm b/FS/FS/cust_main/Billing_Realtime.pm
index 3757ca814..b65860e9b 100644
--- a/FS/FS/cust_main/Billing_Realtime.pm
+++ b/FS/FS/cust_main/Billing_Realtime.pm
@@ -6,6 +6,7 @@ use vars qw( $realtime_bop_decline_quiet ); #ugh
 use Carp;
 use Data::Dumper;
 use Business::CreditCard 0.35;
+use Business::OnlinePayment;
 use FS::UID qw( dbh myconnect );
 use FS::Record qw( qsearch qsearchs );
 use FS::payby;
@@ -14,6 +15,8 @@ use FS::cust_pay_pending;
 use FS::cust_bill_pay;
 use FS::cust_refund;
 use FS::banned_pay;
+use FS::payment_gateway;
+use FS::Misc::Savepoint;
 
 $realtime_bop_decline_quiet = 0;
 
@@ -25,6 +28,7 @@ $me = '[FS::cust_main::Billing_Realtime]';
 
 our $BOP_TESTING = 0;
 our $BOP_TESTING_SUCCESS = 1;
+our $BOP_TESTING_TIMESTAMP = '';
 
 install_callback FS::UID sub { 
   $conf = new FS::Conf;
@@ -112,7 +116,6 @@ I<depend_jobnum> allows payment capture to unlock export jobs
 =cut
 
 # Currently only used by ClientAPI
-# NOT 4.x COMPATIBLE (see below)
 sub realtime_collect {
   my( $self, %options ) = @_;
 
@@ -126,9 +129,12 @@ sub realtime_collect {
   $options{amount} = $self->balance unless exists( $options{amount} );
   return '' unless $options{amount} > 0;
 
-  #### NOT 4.x COMPATIBLE
-  $options{method} = FS::payby->payby2bop($self->payby)
-    unless exists( $options{method} );
+  #huh, in v4, realtime_bop no longer will just process a card without passing
+  # payinfo or cust_payby...
+  if ( ! $options{'payinfo'} && ! $options{'cust_payby'} && $self->has_cust_payby_auto ) {
+    my @cust_payby = $self->cust_payby;
+    $options{'cust_payby'} = $cust_payby[0];
+  }
 
   return $self->realtime_bop({%options});
 
@@ -317,13 +323,16 @@ sub _bop_cust_payby_options {
 
     if ( $cust_payby->locationnum ) {
       my $cust_location = $cust_payby->cust_location;
-      $options->{$_} = $cust_location->$_() for qw( address1 address2 city state zip );
+      $options->{$_} = $cust_location->$_()
+        for qw( address1 address2 city state zip country );
     }
   }
 }
 
 # can be called as class method,
 # but can't load default name/phone fields as class method
+# (why was this added?  ah, it might get called from realtime_tokenize in this
+#  fashion "to tokenize old records on upgrade")
 sub _bop_content {
   my ($self, $options) = @_;
   my %content = ();
@@ -355,14 +364,23 @@ sub _bop_content {
 
   $content{name} = $payname if $payname;
 
-  $content{address} = $options->{'address1'};
-  my $address2 = $options->{'address2'};
-  $content{address} .= ", ". $address2 if length($address2);
+  if ( exists($options->{'address1'}) && length($options->{'address1'}) ) {
+
+    $content{address} = $options->{'address1'};
+    my $address2 = $options->{'address2'};
+    $content{address} .= ", ". $address2 if length($address2);
+
+    $content{$_} = $options->{$_} foreach qw( city state zip country );
+
+  } elsif ( ref($self) ) {
+
+    $content{address} = $self->address1;
+    my $address2 = $self->address2;
+    $content{address} .= ", ". $address2 if length($address2);
+
+    $content{$_} = $self->$_() foreach qw( city state zip country );
 
-  $content{city} = $options->{'city'};
-  $content{state} = $options->{'state'};
-  $content{zip} = $options->{'zip'};
-  $content{country} = $options->{'country'};
+  }
 
   # can't set phone if called as class method
   $content{phone} = $self->daytime || $self->night
@@ -401,7 +419,7 @@ sub realtime_bop {
 
   confess "Can't call realtime_bop within another transaction ".
           '($FS::UID::AutoCommit is false)'
-    unless $FS::UID::AutoCommit;
+    unless $FS::UID::AutoCommit || $BOP_TESTING;
 
   local($DEBUG) = $FS::cust_main::DEBUG if $FS::cust_main::DEBUG > $DEBUG;
 
@@ -417,9 +435,18 @@ sub realtime_bop {
     $options{amount} = $amount;
   }
 
+  return '' unless $options{amount} > 0;
+
   # set fields from passed cust_payby
   _bop_cust_payby_options(\%options);
 
+  # check for banned credit card/ACH
+  my $ban = FS::banned_pay->ban_search(
+    'payby'   => $bop_method2payby{$options{method}},
+    'payinfo' => $options{payinfo},
+  );
+  return "Banned credit card" if $ban && $ban->bantype ne 'warn';
+
   # possibly run a separate transaction to tokenize card number,
   #   so that we never store tokenized card info in cust_pay_pending
   if (($options{method} eq 'CC') && !$self->tokenized($options{'payinfo'})) {
@@ -431,8 +458,6 @@ sub realtime_bop {
       $token_error = $options{'cust_payby'}->replace;
       return $token_error if $token_error;
     }
-    return "Cannot tokenize card info"
-      if $conf->exists('no_saved_cardnumbers') && !$self->tokenized($options{'payinfo'});
   }
 
   ### 
@@ -445,16 +470,24 @@ sub realtime_bop {
     if $conf->config('credit-card-surcharge-percentage', $self->agentnum)
     && $options{method} eq 'CC';
 
+  my $cc_surcharge_flat = 0;
+  $cc_surcharge_flat = $conf->config('credit-card-surcharge-flatfee', $self->agentnum)
+    if $conf->config('credit-card-surcharge-flatfee', $self->agentnum)
+    && $options{method} eq 'CC';
+
   # always add cc surcharge if called from event 
-  if($options{'cc_surcharge_from_event'} && $cc_surcharge_pct > 0) {
-      $cc_surcharge = $options{'amount'} * $cc_surcharge_pct / 100;
+  if($options{'cc_surcharge_from_event'} && ($cc_surcharge_pct > 0 || $cc_surcharge_flat > 0)) {
+    if ($options{'amount'} > 0) {
+      $cc_surcharge = ($options{'amount'} * ($cc_surcharge_pct / 100)) + $cc_surcharge_flat;
       $options{'amount'} += $cc_surcharge;
       $options{'amount'} = sprintf("%.2f", $options{'amount'}); # round (again)?
+    }
   }
-  elsif($cc_surcharge_pct > 0) { # we're called not from event (i.e. from a 
-                                 # payment screen), so consider the given 
-				 # amount as post-surcharge
-    $cc_surcharge = $options{'amount'} - ($options{'amount'} / ( 1 + $cc_surcharge_pct/100 ));
+  elsif($cc_surcharge_pct > 0 || $cc_surcharge_flat > 0) {
+    # we're called not from event (i.e. from a
+    # payment screen), so consider the given
+		# amount as post-surcharge-processing_fee
+    $cc_surcharge = $options{'amount'} - $options{'processing-fee'} - (($options{'amount'} - ($cc_surcharge_flat + $options{'processing-fee'})) / ( 1 + $cc_surcharge_pct/100 )) if $options{'amount'} > 0;
   }
   
   $cc_surcharge = sprintf("%.2f",$cc_surcharge) if $cc_surcharge > 0;
@@ -508,16 +541,6 @@ sub realtime_bop {
   die $@ if $@;
 
   ###
-  # check for banned credit card/ACH
-  ###
-
-  my $ban = FS::banned_pay->ban_search(
-    'payby'   => $bop_method2payby{$options{method}},
-    'payinfo' => $options{payinfo},
-  );
-  return "Banned credit card" if $ban && $ban->bantype ne 'warn';
-
-  ###
   # check for term discount validity
   ###
 
@@ -673,7 +696,7 @@ sub realtime_bop {
   my $cust_pay_pending = new FS::cust_pay_pending {
     'custnum'           => $self->custnum,
     'paid'              => $options{amount},
-    '_date'             => '',
+    '_date'             => $BOP_TESTING ? $BOP_TESTING_TIMESTAMP : '',
     'payby'             => $bop_method2payby{$options{method}},
     'payinfo'           => $options{payinfo},
     'paymask'           => $options{paymask},
@@ -748,7 +771,7 @@ sub realtime_bop {
     return { reference => $cust_pay_pending->paypendingnum,
              map { $_ => $transaction->$_ } qw ( popup_url collectitems ) };
 
-  } elsif ( $transaction->is_success() && $action2 ) {
+  } elsif ( !$BOP_TESTING && $transaction->is_success() && $action2 ) {
 
     $cust_pay_pending->status('authorized');
     my $cpp_authorized_err = $cust_pay_pending->replace;
@@ -937,7 +960,7 @@ sub _realtime_bop_result {
        'custnum'  => $self->custnum,
        'invnum'   => $options{'invnum'},
        'paid'     => $cust_pay_pending->paid,
-       '_date'    => '',
+       '_date'    => $BOP_TESTING ? $BOP_TESTING_TIMESTAMP : '',
        'payby'    => $cust_pay_pending->payby,
        'payinfo'  => $options{'payinfo'},
        'paymask'  => $options{'paymask'} || $cust_pay_pending->paymask,
@@ -958,21 +981,29 @@ sub _realtime_bop_result {
     local $FS::UID::AutoCommit = 0;
     my $dbh = dbh;
 
-    #start a transaction, insert the cust_pay and set cust_pay_pending.status to done in a single transction
+    my $savepoint_label = '_realtime_bop_result';
+    savepoint_create( $savepoint_label );
 
-    my $error = $cust_pay->insert($options{'manual'} ? ( 'manual' => 1 ) : () );
+    #start a transaction, insert the cust_pay and set cust_pay_pending.status to done in a single transction
+    my $error = $cust_pay->insert(
+      $options{'manual'} ? ( 'manual' => 1 ) : (),
+      $options{'processing-fee'} > 0 ? ( 'processing-fee' => $options{'processing-fee'} ) : (),
+    );
 
     if ( $error ) {
-      $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
+      savepoint_rollback( $savepoint_label );
+
       $cust_pay->invnum(''); #try again with no specific invnum
       $cust_pay->paynum('');
-      my $error2 = $cust_pay->insert( $options{'manual'} ?
-                                      ( 'manual' => 1 ) : ()
-                                    );
+      my $error2 = $cust_pay->insert(
+        $options{'manual'} ? ( 'manual' => 1 ) : (),
+        $options{'processing-fee'} > 0 ? ( 'processing-fee' => $options{'processing-fee'} ) : (),
+      );
       if ( $error2 ) {
         # gah.  but at least we have a record of the state we had to abort in
         # from cust_pay_pending now.
-        $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
+        savepoint_rollback_and_release( $savepoint_label );
+
         my $e = "WARNING: $options{method} captured but payment not recorded -".
                 " error inserting payment (". $payment_gateway->gateway_module.
                 "): $error2".
@@ -987,9 +1018,10 @@ sub _realtime_bop_result {
     my $jobnum = $cust_pay_pending->jobnum;
     if ( $jobnum ) {
        my $placeholder = qsearchs( 'queue', { 'jobnum' => $jobnum } );
-      
+
        unless ( $placeholder ) {
-         $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
+         savepoint_rollback_and_release( $savepoint_label );
+
          my $e = "WARNING: $options{method} captured but job $jobnum not ".
              "found for paypendingnum ". $cust_pay_pending->paypendingnum. "\n";
          warn $e;
@@ -999,7 +1031,8 @@ sub _realtime_bop_result {
        $error = $placeholder->delete;
 
        if ( $error ) {
-         $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
+        savepoint_rollback_and_release( $savepoint_label );
+
          my $e = "WARNING: $options{method} captured but could not delete ".
               "job $jobnum for paypendingnum ".
               $cust_pay_pending->paypendingnum. ": $error\n";
@@ -1021,8 +1054,8 @@ sub _realtime_bop_result {
     my $cpp_done_err = $cust_pay_pending->replace;
 
     if ( $cpp_done_err ) {
+      savepoint_rollback_and_release( $savepoint_label );
 
-      $dbh->rollback or die $dbh->errstr if $oldAutoCommit;
       my $e = "WARNING: $options{method} captured but payment not recorded - ".
               "error updating status for paypendingnum ".
               $cust_pay_pending->paypendingnum. ": $cpp_done_err \n";
@@ -1030,7 +1063,7 @@ sub _realtime_bop_result {
       return $e;
 
     } else {
-
+      savepoint_release( $savepoint_label );
       $dbh->commit or die $dbh->errstr if $oldAutoCommit;
 
       if ( $options{'apply'} ) {
@@ -1043,7 +1076,7 @@ sub _realtime_bop_result {
       }
 
       # have a CC surcharge portion --> one-time charge
-      if ( $options{'cc_surcharge'} > 0 ) { 
+      if ( $options{'cc_surcharge'} > 0 || $options{'processing-fee'} > 0) {
 	    # XXX: this whole block needs to be in a transaction?
 
 	  my $invnum;
@@ -1064,42 +1097,119 @@ sub _realtime_bop_result {
 	  unless ( $invnum ) {
 	    # XXX: unlikely case - pre-paying before any invoices generated
 	    # what it should do is create a new invoice and pick it
-		warn 'CC SURCHARGE AND NO INVOICES PICKED TO APPLY IT!';
+		warn 'CC SURCHARGE OR PROCESS FEE AND NO INVOICES PICKED TO APPLY IT!';
 		return '';
 	  }
 
-	  my $cust_pkg;
-	  my $charge_error = $self->charge({
+    if ($options{'cc_surcharge'} > 0) {
+	    my $cust_pkg;
+      my $cc_surcharge_text = 'Credit Card Surcharge';
+      $cc_surcharge_text = $conf->config('credit-card-surcharge-text', $self->agentnum) if $conf->exists('credit-card-surcharge-text', $self->agentnum);
+	    my $charge_error = $self->charge({
 				    'amount' 	=> $options{'cc_surcharge'},
-				    'pkg' 	=> 'Credit Card Surcharge',
+				    'pkg' 	=> $cc_surcharge_text,
 				    'setuptax'  => 'Y',
 				    'cust_pkg_ref' => \$cust_pkg,
-				});
-	  if($charge_error) {
-		warn 'Unable to add CC surcharge cust_pkg';
-		return '';
-	  }
+			});
+
+	    if($charge_error) {
+		    warn 'Unable to add CC surcharge cust_pkg';
+		    return '';
+	    }
+
+      $cust_pkg->setup(time);
+      my $cp_error = $cust_pkg->replace;
+      if($cp_error) {
+        warn 'Unable to set setup time on cust_pkg for cc surcharge';
+        # but keep going...
+      }
 
-	  $cust_pkg->setup(time);
-	  my $cp_error = $cust_pkg->replace;
-	  if($cp_error) {
-	      warn 'Unable to set setup time on cust_pkg for cc surcharge';
-	    # but keep going...
-	  }
-				    
-	  my $cust_bill = qsearchs('cust_bill', { 'invnum' => $invnum });
-	  unless ( $cust_bill ) {
-	      warn "race condition + invoice deletion just happened";
-	      return '';
-	  }
+      my $cust_bill = qsearchs('cust_bill', { 'invnum' => $invnum });
+      unless ( $cust_bill ) {
+        warn "race condition + invoice deletion just happened";
+        return '';
+      }
 
-	  my $grand_error = 
-	    $cust_bill->add_cc_surcharge($cust_pkg->pkgnum,$options{'cc_surcharge'});
+      my $grand_error =
+        $cust_bill->add_cc_surcharge($cust_pkg->pkgnum,$options{'cc_surcharge'});
 
-	  warn "cannot add CC surcharge to invoice #$invnum: $grand_error"
-	    if $grand_error;
+      warn "cannot add CC surcharge to invoice #$invnum: $grand_error"
+        if $grand_error;
+    } # end if $options{'cc_surcharge'}
+
+    if ($options{'processing-fee'} > 0) {
+      my $pf_cust_pkg;
+      my $processing_fee_text = 'Payment Processing Fee';
+
+      my $conf = new FS::Conf;
+
+      my $pf_seperate_bill;
+      my $pf_bill_now;
+      if ($conf->exists('processing-fee_on_separate_invoice')) {
+        $pf_seperate_bill = 'Y';
+        $pf_bill_now = '1';
+      }
+
+      my $pf_change_error = $self->charge({
+            'amount'  => $options{'processing-fee'},
+            'pkg'   => $processing_fee_text,
+            'setuptax'  => 'Y',
+            'cust_pkg_ref' => \$pf_cust_pkg,
+            'separate_bill' => $pf_seperate_bill,
+            'bill_now' => $pf_bill_now,
+      });
+
+      if($pf_change_error) {
+        warn 'Unable to add payment processing fee';
+        return '';
+      }
+
+      $pf_cust_pkg->setup(time);
+      my $pf_error = $pf_cust_pkg->replace;
+      if($pf_error) {
+        warn 'Unable to set setup time on cust_pkg for processing fee';
+        # but keep going...
       }
 
+      if ($conf->exists('processing-fee_on_separate_invoice')) {
+        my $cust_bill_pkg = qsearchs( 'cust_bill_pkg', { 'pkgnum' => $pf_cust_pkg->pkgnum } );
+
+        my $pf_cust_bill = qsearchs('cust_bill', { 'invnum' => $cust_bill_pkg->invnum });
+        unless ( $pf_cust_bill ) {
+          warn "no processing fee inv found!";
+          return '';
+        }
+
+        my $pf_apply_error = $pf_cust_bill->apply_payments_and_credits;
+
+        my $cust_bill = qsearchs('cust_bill', { 'invnum' => $invnum });
+        unless ( $cust_bill ) {
+          warn "race condition + invoice deletion just happened";
+         return '';
+        }
+
+        my $grand_pf_error = $cust_bill->apply_payments_and_credits;
+
+        warn "cannot apply Processing fee to invoice #$invnum: $grand_pf_error - $pf_apply_error"
+          if $grand_pf_error || $pf_apply_error;
+      } ## processing-fee_on_separate_invoice
+      else {
+        my $cust_bill = qsearchs('cust_bill', { 'invnum' => $invnum });
+        unless ( $cust_bill ) {
+          warn "race condition + invoice deletion just happened";
+          return '';
+        }
+
+        my $grand_pf_error =
+          $cust_bill->add_cc_surcharge($pf_cust_pkg->pkgnum,$options{'processing-fee'});
+
+        warn "cannot add Processing fee to invoice #$invnum: $grand_pf_error"
+          if $grand_pf_error;
+      } ## no processing-fee_on_separate_invoice
+    } #end if $options{'processing-fee'}
+
+      } #end if ( $options{'cc_surcharge'} > 0 || $options{'processing-fee'} > 0)
+
       return ''; #no error
 
     }
@@ -1194,6 +1304,7 @@ sub _realtime_bop_result {
               "resolved - error updating status for paypendingnum ".
               $cust_pay_pending->paypendingnum. ": $cpp_done_err \n";
       warn $e;
+      #XXX internal system log $e (what's going on?)
       $perror = "$e ($perror)";
     }
 
@@ -1460,9 +1571,10 @@ sub realtime_refund_bop {
       ( $gatewaynum, $processor, $auth, $order_number ) = ( $2, $3, $4, $6 );
     }
 
+    my $payment_gateway;
     if ( $gatewaynum ) { #gateway for the payment to be refunded
 
-      my $payment_gateway =
+      $payment_gateway =
         qsearchs('payment_gateway', { 'gatewaynum' => $gatewaynum } );
       die "payment gateway $gatewaynum not found"
         unless $payment_gateway;
@@ -1476,7 +1588,7 @@ sub realtime_refund_bop {
     } else { #try the default gateway
 
       my $conf_processor;
-      my $payment_gateway =
+      $payment_gateway =
         $self->agent->payment_gateway('method' => $options{method});
 
       ( $conf_processor, $login, $password, $namespace ) =
@@ -1486,22 +1598,41 @@ sub realtime_refund_bop {
       @bop_options = $payment_gateway->gatewaynum
                        ? $payment_gateway->options
                        : @{ $payment_gateway->get('options') };
+      my %bop_options = @bop_options;
 
       return "processor of payment $options{'paynum'} $processor does not".
              " match default processor $conf_processor"
-        unless $processor eq $conf_processor;
+        unless ($processor eq $conf_processor)
+            || (($conf_processor eq 'CardFortress') && ($processor eq $bop_options{'gateway'}));
+
+      $processor = $conf_processor;
 
     }
 
+    # if gateway has switched to CardFortress but token_check hasn't run yet,
+    # tokenize just this record now, so that token gets passed/set appropriately
+    if ($cust_pay->payby eq 'CARD' && !$cust_pay->tokenized) {
+      my %tokenopts = (
+        'payment_gateway' => $payment_gateway,
+        'method'          => 'CC',
+        'payinfo'         => $cust_pay->payinfo,
+        'paydate'         => $cust_pay->paydate,
+      );
+      my $error = $self->realtime_tokenize(\%tokenopts); # no-op unless gateway can tokenize
+      if ($self->tokenized($tokenopts{'payinfo'})) { # implies no error
+        warn "  tokenizing cust_pay\n" if $DEBUG > 1;
+        $cust_pay->payinfo($tokenopts{'payinfo'});
+        $error = $cust_pay->replace;
+      }
+      return $error if $error;
+    }
 
   } else { # didn't specify a paynum, so look for agent gateway overrides
            # like a normal transaction 
  
     my $payment_gateway =
-      $self->agent->payment_gateway( 'method'  => $options{method},
-                                     #'payinfo' => $payinfo,
-                                   );
-    my( $processor, $login, $password, $namespace ) =
+      $self->agent->payment_gateway( 'method'  => $options{method} );
+    ( $processor, $login, $password, $namespace ) =
       map { my $method = "gateway_$_"; $payment_gateway->$method }
         qw( module username password namespace );
 
@@ -1633,18 +1764,22 @@ sub realtime_refund_bop {
     if length($payip);
 
   my $payinfo = '';
+  my $paymask = ''; # for refund record
   if ( $options{method} eq 'CC' ) {
 
     if ( $cust_pay ) {
       $content{card_number} = $payinfo = $cust_pay->payinfo;
+      $paymask = $cust_pay->paymask;
       (exists($options{'paydate'}) ? $options{'paydate'} : $cust_pay->paydate)
         =~ /^\d{2}(\d{2})[\/\-](\d+)[\/\-]\d+$/ &&
         ($content{expiration} = "$2/$1");  # where available
     } else {
-      $content{card_number} = $payinfo = $self->payinfo;
-      (exists($options{'paydate'}) ? $options{'paydate'} : $self->paydate)
-        =~ /^\d{2}(\d{2})[\/\-](\d+)[\/\-]\d+$/;
-      $content{expiration} = "$2/$1";
+      # this really needs a better cleanup
+      die "Refund without paynum not supported";
+#      $content{card_number} = $payinfo = $self->payinfo;
+#      (exists($options{'paydate'}) ? $options{'paydate'} : $self->paydate)
+#        =~ /^\d{2}(\d{2})[\/\-](\d+)[\/\-]\d+$/;
+#      $content{expiration} = "$2/$1";
     }
 
   } elsif ( $options{method} eq 'ECHECK' ) {
@@ -1708,6 +1843,7 @@ sub realtime_refund_bop {
     '_date'    => '',
     'payby'    => $bop_method2payby{$options{method}},
     'payinfo'  => $payinfo,
+    'paymask'  => $paymask,
     'reasonnum'     => $options{'reasonnum'},
     'gatewaynum'    => $gatewaynum, # may be null
     'processor'     => $processor,
@@ -1774,6 +1910,13 @@ sub realtime_verify_bop {
   return "No cust_payby" unless $options{'cust_payby'};
   _bop_cust_payby_options(\%options);
 
+  # check for banned credit card/ACH
+  my $ban = FS::banned_pay->ban_search(
+    'payby'   => $bop_method2payby{'CC'},
+    'payinfo' => $options{payinfo},
+  );
+  return "Banned credit card" if $ban && $ban->bantype ne 'warn';
+
   # possibly run a separate transaction to tokenize card number,
   #   so that we never store tokenized card info in cust_pay_pending
   if (($options{method} eq 'CC') && !$self->tokenized($options{'payinfo'})) {
@@ -1781,8 +1924,6 @@ sub realtime_verify_bop {
     return $token_error if $token_error;
     #important that we not replace cust_payby here,
     #because cust_payby->replace uses realtime_verify_bop!
-    return "Cannot tokenize card info"
-      if $conf->exists('no_saved_cardnumbers') && !$self->tokenized($options{'payinfo'});
   }
 
   ###
@@ -1796,16 +1937,6 @@ sub realtime_verify_bop {
   die $@ if $@;
 
   ###
-  # check for banned credit card/ACH
-  ###
-
-  my $ban = FS::banned_pay->ban_search(
-    'payby'   => $bop_method2payby{'CC'},
-    'payinfo' => $options{payinfo},
-  );
-  return "Banned credit card" if $ban && $ban->bantype ne 'warn';
-
-  ###
   # massage data
   ###
 
@@ -2213,6 +2344,13 @@ sub realtime_tokenize {
   return '' unless $options{method} eq 'CC';
   return '' if $self->tokenized($options{payinfo}); #already tokenized
 
+  # check for banned credit card/ACH
+  my $ban = FS::banned_pay->ban_search(
+    'payby'   => $bop_method2payby{'CC'},
+    'payinfo' => $options{payinfo},
+  );
+  return "Banned credit card" if $ban && $ban->bantype ne 'warn';
+
   ###
   # select a gateway
   ###
@@ -2241,16 +2379,6 @@ sub realtime_tokenize {
                 && grep /^Tokenize$/, @{$supported_actions{'CC'}};
 
   ###
-  # check for banned credit card/ACH
-  ###
-
-  my $ban = FS::banned_pay->ban_search(
-    'payby'   => $bop_method2payby{'CC'},
-    'payinfo' => $options{payinfo},
-  );
-  return "Banned credit card" if $ban && $ban->bantype ne 'warn';
-
-  ###
   # massage data
   ###
 
@@ -2297,7 +2425,7 @@ sub realtime_tokenize {
     'type'           => 'CC',
     _bop_auth(\%options),          
     'action'         => 'Tokenize',
-    'description'    => $options{'description'}
+    'description'    => $options{'description'},
     %$bop_content,
     %content, #after
   );
@@ -2347,7 +2475,7 @@ sub tokenized {
   FS::cust_pay->tokenized($payinfo);
 }
 
-=item token_check
+=item token_check [ quiet => 1, queue => 1, daily => 1 ]
 
 NOT A METHOD.  Acts on all customers.  Placed here because it makes
 use of module-internal methods, and to keep everything that uses
@@ -2356,74 +2484,149 @@ Billing::OnlinePayment all in one place.
 Tokenizes all tokenizable card numbers from payinfo in cust_payby and 
 CARD transactions in cust_pay_pending, cust_pay, cust_pay_void and cust_refund.
 
-If all configured gateways have the ability to tokenize, then detection of
-an untokenizable record will cause a fatal error.
+If the I<queue> flag is set, newly tokenized records will be immediately
+committed, regardless of AutoCommit, so as to release the mutex on the record.
+
+If all configured gateways have the ability to tokenize, detection of an 
+untokenizable record will cause a fatal error.  However, if the I<queue> flag 
+is set, this will instead cause a critical error to be recorded in the log, 
+and any other tokenizable records will still be committed.
+
+If the I<daily> flag is also set, detection of existing untokenized records will 
+record an info message in the system log (because they should have never appeared 
+in the first place.)  Tokenization will still be attempted.
+
+If any configured gateways do NOT have the ability to tokenize, or if a
+default gateway is not configured, then untokenized records are not considered 
+a threat, and no critical errors will be generated in the log.
 
 =cut
 
 sub token_check {
-  # no input, acts on all customers
+  #acts on all customers
+  my %opt = @_;
+  my $debug = !$opt{'quiet'} || $DEBUG;
+  my $hascritical = 0;
 
-  eval "use FS::Cursor";  
-  return "Error initializing FS::Cursor: ".$@ if $@;
+  warn "token_check called with opts\n".Dumper(\%opt) if $debug;
 
-  my $dbh = dbh;
+  # force some explicitness when invoking this method
+  die "token_check must run with queue flag if run with daily flag"
+    if $opt{'daily'} && !$opt{'queue'};
+
+  my $conf = FS::Conf->new;
+
+  my $log = FS::Log->new('FS::cust_main::Billing_Realtime::token_check');
 
-  # get list of all gateways in table (not counting default gateway)
   my $cache = {}; #cache for module info
-  my $sth = $dbh->prepare('SELECT DISTINCT gatewaynum FROM payment_gateway')
-    or die $dbh->errstr;
-  $sth->execute or die $sth->errstr;
-  my @gatewaynums;
-  while (my $row = $sth->fetchrow_hashref) {
-    push(@gatewaynums,$row->{'gatewaynum'});
-  }
-  $sth->finish;
 
-  # look for a gateway that can't tokenize
-  my $disallow_untokenized = 1;
-  foreach my $gatewaynum ('',@gatewaynums) {
-    my $gateway = FS::agent->payment_gateway( load_gatewaynum => $gatewaynum, nofatal => 1 );
-    if (!$gateway) { # already died if $gatewaynum
+  # look for a gateway that can and can't tokenize
+  my $require_tokenized = 1;
+  my $someone_tokenizing = 0;
+  foreach my $gateway (
+    FS::payment_gateway->all_gateways(
+      'method'  => 'CC',
+      'conf'    => $conf,
+      'nofatal' => 1,
+    )
+  ) {
+    if (!$gateway) {
       # no default gateway, no promise to tokenize
       # can just load other gateways as-needeed below
-      $disallow_untokenized = 0;
-      last;
+      $require_tokenized = 0;
+      last if $someone_tokenizing;
+      next;
     }
     my $info = _token_check_gateway_info($cache,$gateway);
-    return $info unless ref($info); # means it's an error message
-    unless ($info->{'can_tokenize'}) {
+    die $info unless ref($info); # means it's an error message
+    if ($info->{'can_tokenize'}) {
+      $someone_tokenizing = 1;
+    } else {
       # a configured gateway can't tokenize, that's all we need to know right now
       # can just load other gateways as-needeed below
-      $disallow_untokenized = 0;
-      last;
+      $require_tokenized = 0;
+      last if $someone_tokenizing;
     }
   }
 
+  unless ($someone_tokenizing) { #no need to check, if no one can tokenize
+    warn "no gateways tokenize\n" if $debug;
+    return;
+  }
+
+  warn "REQUIRE TOKENIZED" if $require_tokenized && $debug;
+
+  # upgrade does not call this with autocommit turned on,
+  # and autocommit will be ignored if opt queue is set,
+  # but might as well be thorough...
   my $oldAutoCommit = $FS::UID::AutoCommit;
   local $FS::UID::AutoCommit = 0;
+  my $dbh = dbh;
+
+  # for retrieving data in chunks
+  my $step = 500;
+  my $offset = 0;
 
   ### Tokenize cust_payby
 
-  my $cust_search = FS::Cursor->new({ table => 'cust_main' },$dbh);
-  while (my $cust_main = $cust_search->fetch) {
+  my @recnums;
+
+CUSTLOOP:
+  while (my $custnum = _token_check_next_recnum($dbh,'cust_main',$step,\$offset,\@recnums)) {
+    my $cust_main = FS::cust_main->by_key($custnum);
+    my $payment_gateway;
     foreach my $cust_payby ($cust_main->cust_payby('CARD','DCRD')) {
-      next if $cust_payby->tokenized;
-      # load gateway first, just so we can cache it
-      my $payment_gateway = $cust_main->_payment_gateway({
-        'nofatal' => 1, # handle error smoothly below
+
+      # see if it's already tokenized
+      if ($cust_payby->tokenized) {
+        warn "cust_payby ".$cust_payby->get($cust_payby->primary_key)." already tokenized" if $debug;
+        next;
+      }
+
+      if ($require_tokenized && $opt{'daily'}) {
+        $log->info("Untokenized card number detected in cust_payby ".$cust_payby->custpaybynum. '; tokenizing');
+        $dbh->commit or die $dbh->errstr; # commit log message
+      }
+
+      # only load gateway if we need to, and only need to load it once
+      $payment_gateway ||= $cust_main->_payment_gateway({
+        'method'  => 'CC',
+        'conf'    => $conf,
+        'nofatal' => 1, # handle lack of gateway smoothly below
       });
       unless ($payment_gateway) {
         # no reason to have untokenized card numbers saved if no gateway,
-        #   but only fatal if we expected everyone to tokenize card numbers
-        next unless $disallow_untokenized;
-        $cust_search->DESTROY;
+        #   but only a problem if we expected everyone to tokenize card numbers
+        unless ($require_tokenized) {
+          warn "Skipping cust_payby for cust_main ".$cust_main->custnum.", no payment gateway" if $debug;
+          next CUSTLOOP; # can skip rest of customer
+        }
+        my $error = "No gateway found for custnum ".$cust_main->custnum;
+        if ($opt{'queue'}) {
+          $hascritical = 1;
+          $log->critical($error);
+          $dbh->commit or die $dbh->errstr; # commit error message
+          next; # not next CUSTLOOP, want to record error for every cust_payby
+        }
         $dbh->rollback if $oldAutoCommit;
-        return "No gateway found for custnum ".$cust_main->custnum;
+        die $error;
       }
+
       my $info = _token_check_gateway_info($cache,$payment_gateway);
+      unless (ref($info)) {
+        # only throws error if Business::OnlinePayment won't load,
+        #   which is just cause to abort this whole process, even if queue
+        $dbh->rollback if $oldAutoCommit;
+        die $info; # error message
+      }
       # no fail here--a configured gateway can't tokenize, so be it
-      next unless ref($info) && $info->{'can_tokenize'};
+      unless ($info->{'can_tokenize'}) {
+        warn "Skipping ".$cust_main->custnum." cannot tokenize" if $debug;
+        next;
+      }
+
+      # time to tokenize
+      $cust_payby = $cust_payby->select_for_update;
       my %tokenopts = (
         'payment_gateway' => $payment_gateway,
         'cust_payby'      => $cust_payby,
@@ -2435,11 +2638,21 @@ sub token_check {
         $error ||= 'Unknown error';
       }
       if ($error) {
-        $cust_search->DESTROY;
+        $error = "Error tokenizing cust_payby ".$cust_payby->custpaybynum.": ".$error;
+        if ($opt{'queue'}) {
+          $hascritical = 1;
+          $log->critical($error);
+          $dbh->commit or die $dbh->errstr; # commit log message, release mutex
+          next; # not next CUSTLOOP, want to record error for every cust_payby
+        }
         $dbh->rollback if $oldAutoCommit;
-        return "Error tokenizing cust_payby ".$cust_payby->custpaybynum.": ".$error;
+        die $error;
       }
+      $dbh->commit or die $dbh->errstr if $opt{'queue'}; # release mutex
+      warn "TOKENIZED cust_payby ".$cust_payby->get($cust_payby->primary_key) if $debug;
     }
+    warn "cust_payby upgraded for custnum ".$cust_main->custnum if $debug;
+
   }
 
   ### Tokenize/mask transaction tables
@@ -2450,50 +2663,132 @@ sub token_check {
   # grep assistance:
   #   $cust_pay_pending->replace, $cust_pay->replace, $cust_pay_void->replace, $cust_refund->replace all run here
   foreach my $table ( qw(cust_pay_pending cust_pay cust_pay_void cust_refund) ) {
-    my $search = FS::Cursor->new({
-      table     => $table,
-      hashref   => { 'payby' => 'CARD' },
-    },$dbh);
-    while (my $record = $search->fetch) {
-      next if $record->tokenized;
-      next if !$record->payinfo; #shouldn't happen, but at least it's not a card number
-      next if $record->payinfo =~ /N\/A/; # ??? Not sure why we do this, but it's not a card number
-
-      # don't use customer agent gateway here, use the gatewaynum specified by the record
-      my $gatewaynum = $record->gatewaynum || '';
-      my $gateway = FS::agent->payment_gateway( load_gatewaynum => $gatewaynum );
-      unless ($gateway) { # already died if $gatewaynum
-        # only fatal if we expected everyone to tokenize
-        next unless $disallow_untokenized;
-        $search->DESTROY;
-        $dbh->rollback if $oldAutoCommit;
-        return "No gateway found for $table ".$record->get($record->primary_key);
+    warn "Checking $table" if $debug;
+
+    # FS::Cursor does not seem to work over multiple commits (gives cursor not found errors)
+    # loading only record ids, then loading individual records one at a time
+    my $tclass = 'FS::'.$table;
+    $offset = 0;
+    @recnums = ();
+
+    while (my $recnum = _token_check_next_recnum($dbh,$table,$step,\$offset,\@recnums)) {
+      my $record = $tclass->by_key($recnum);
+      unless ($record->payby eq 'CARD') {
+        warn "Skipping non-card record for $table ".$record->get($record->primary_key) if $debug;
+        next;
       }
-      my $info = _token_check_gateway_info($cache,$gateway);
-      unless (ref($info)) {
-        # only throws error if Business::OnlinePayment won't load,
-        #   which is just cause to abort this whole process
-        $search->DESTROY;
-        $dbh->rollback if $oldAutoCommit;
-        return $info; # error message
+      if (FS::cust_main::Billing_Realtime->tokenized($record->payinfo)) {
+        warn "Skipping tokenized record for $table ".$record->get($record->primary_key) if $debug;
+        next;
+      }
+      if (!$record->payinfo) { #shouldn't happen, but at least it's not a card number
+        warn "Skipping blank payinfo for $table ".$record->get($record->primary_key) if $debug;
+        next;
+      }
+      if ($record->payinfo =~ /N\/A/) { # ??? Not sure why we do this, but it's not a card number
+        warn "Skipping NA payinfo for $table ".$record->get($record->primary_key) if $debug;
+        next;
       }
 
-      # a configured gateway can't tokenize, move along
-      next unless $info->{'can_tokenize'};
+      if ($require_tokenized && $opt{'daily'}) {
+        $log->info("Untokenized card number detected in $table ".$record->get($record->primary_key). ';tokenizing');
+        $dbh->commit or die $dbh->errstr; # commit log message
+      }
 
       my $cust_main = $record->cust_main;
-      unless ($cust_main || (
+      if (!$cust_main) {
         # might happen for cust_pay_pending from failed verify records,
         #   in which case we attempt tokenization without cust_main
         # everything else should absolutely have a cust_main
-        $table eq 'cust_pay_pending'
-          && $record->{'custnum_pending'}
-          && !$disallow_untokenized
-      )) {
-        $search->DESTROY;
+        if ($table eq 'cust_pay_pending' and !$record->custnum ) {
+          # override the usual safety check and allow the record to be
+          # updated even without a custnum.
+          $record->set('custnum_pending', 1);
+        } else {
+          my $error = "Could not load cust_main for $table ".$record->get($record->primary_key);
+          if ($opt{'queue'}) {
+            $hascritical = 1;
+            $log->critical($error);
+            $dbh->commit or die $dbh->errstr; # commit log message
+            next;
+          }
+          $dbh->rollback if $oldAutoCommit;
+          die $error;
+        }
+      }
+
+      my $gateway;
+
+      # use the gatewaynum specified by the record if possible
+      $gateway = FS::payment_gateway->by_key_with_namespace(
+        'gatewaynum' => $record->gatewaynum,
+      ) if $record->gateway;
+
+      # otherwise use the cust agent gateway if possible (which realtime_refund_bop would do)
+      # otherwise just use default gateway
+      unless ($gateway) {
+
+        $gateway = $cust_main 
+                 ? $cust_main->agent->payment_gateway
+                 : FS::payment_gateway->default_gateway;
+
+        # check for processor mismatch
+        unless ($table eq 'cust_pay_pending') { # has no processor table
+          if (my $processor = $record->processor) {
+
+            my $conf_processor = $gateway->gateway_module;
+            my %bop_options = $gateway->gatewaynum
+                            ? $gateway->options
+                            : @{ $gateway->get('options') };
+
+            # this is the same standard used by realtime_refund_bop
+            unless (
+              ($processor eq $conf_processor) ||
+              (($conf_processor eq 'CardFortress') && ($processor eq $bop_options{'gateway'}))
+            ) {
+
+              # processors don't match, so refund already cannot be run on this object,
+              # regardless of what we do now...
+              # but unless we gotta tokenize everything, just leave well enough alone
+              unless ($require_tokenized) {
+                warn "Skipping mismatched processor for $table ".$record->get($record->primary_key) if $debug;
+                next;
+              }
+              ### no error--we'll tokenize using the new gateway, just to remove stored payinfo,
+              ### because refunds are already impossible for this record, anyway
+
+            } # end processor mismatch
+
+          } # end record has processor
+        } # end not cust_pay_pending
+
+      }
+
+      # means no default gateway, no promise to tokenize, can skip
+      unless ($gateway) {
+        warn "Skipping missing gateway for $table ".$record->get($record->primary_key) if $debug;
+        next;
+      }
+
+      my $info = _token_check_gateway_info($cache,$gateway);
+      unless (ref($info)) {
+        # only throws error if Business::OnlinePayment won't load,
+        #   which is just cause to abort this whole process, even if queue
         $dbh->rollback if $oldAutoCommit;
-        return "Could not load cust_main for $table ".$record->get($record->primary_key);
+        die $info; # error message
+      }
+
+      # a configured gateway can't tokenize, move along
+      unless ($info->{'can_tokenize'}) {
+        warn "Skipping, cannot tokenize $table ".$record->get($record->primary_key) if $debug;
+        next;
       }
+
+      warn "ATTEMPTING GATEWAY-ONLY TOKENIZE" if $debug && !$cust_main;
+
+      # if we got this far, time to mutex
+      $record->select_for_update;
+
       # no clear record of name/address/etc used for transaction,
       # but will load name/phone/id from customer if run as an object method,
       # so we try that if we can
@@ -2513,16 +2808,49 @@ sub token_check {
         $error ||= 'Unknown error';
       }
       if ($error) {
-        $search->DESTROY;
+        $error = "Error tokenizing $table ".$record->get($record->primary_key).": ".$error;
+        if ($opt{'queue'}) {
+          $hascritical = 1;
+          $log->critical($error);
+          $dbh->commit or die $dbh->errstr; # commit log message, release mutex
+          next;
+        }
         $dbh->rollback if $oldAutoCommit;
-        return "Error tokenizing $table ".$record->get($record->primary_key).": ".$error;
+        die $error;
       }
+      $dbh->commit or die $dbh->errstr if $opt{'queue'}; # release mutex
+      warn "TOKENIZED $table ".$record->get($record->primary_key) if $debug;
+
     } # end record loop
   } # end table loop
 
-  $dbh->commit if $oldAutoCommit;
+  $dbh->commit or die $dbh->errstr if $oldAutoCommit;
 
-  return '';
+  return $hascritical ? 'Critical errors occurred on some records, see system log' : '';
+}
+
+# not a method!
+sub _token_check_next_recnum {
+  my ($dbh,$table,$step,$offset,$recnums) = @_;
+  my $recnum = shift @$recnums;
+  return $recnum if $recnum;
+  my $tclass = 'FS::'.$table;
+  my $sth = $dbh->prepare(
+    'SELECT '.$tclass->primary_key.
+    ' FROM '.$table.
+    " WHERE ( is_tokenized IS NULL OR is_tokenized = '' ) ".
+    ' ORDER BY '.$tclass->primary_key.
+    ' LIMIT '.$step.
+    ' OFFSET '.$$offset
+  ) or die $dbh->errstr;
+  $sth->execute() or die $sth->errstr;
+  my @recnums;
+  while (my $rec = $sth->fetchrow_hashref) {
+    push @$recnums, $rec->{$tclass->primary_key};
+  }
+  $sth->finish();
+  $$offset += $step;
+  return shift @$recnums;
 }
 
 # not a method!
@@ -2563,8 +2891,6 @@ sub _token_check_gateway_info {
   $info->{'void_requires_card'} = 1
     if $transaction->info('CC_void_requires_card');
 
-  $cache->{$payment_gateway->gateway_module} = $info;
-
   return $info;
 }