71513: Card tokenization [upgrade implemented]

12 files changed, 431 insertions, 151 deletions

diff --git a/FS/FS/Cron/cleanup.pm b/FS/FS/Cron/cleanup.pm
index 6ec4013..9d0c067 100644
--- a/FS/FS/Cron/cleanup.pm
+++ b/FS/FS/Cron/cleanup.pm
@@ -8,12 +8,26 @@ use FS::Record qw( qsearch );
 
 # start janitor jobs
 sub cleanup {
-# fix locations that are missing coordinates
+  my %opt = @_;
+
+  # fix locations that are missing coordinates
   my $job = FS::queue->new({
       'job'     => 'FS::cust_location::process_set_coord',
       'status'  => 'new'
   });
   $job->insert('_JOB');
+
+  # check card number tokenization
+  $job = FS::queue->new({
+      'job'     => 'FS::cust_main::Billing_Realtime::token_check',
+      'status'  => 'new'
+  });
+  $job->insert(
+    %opt,
+    'queue' => 1,
+    'daily' => 1,
+  );
+
 }
 
 sub cleanup_before_backup {
diff --git a/FS/FS/Upgrade.pm b/FS/FS/Upgrade.pm
index 7fbbbaa..31311e9 100644
--- a/FS/FS/Upgrade.pm
+++ b/FS/FS/Upgrade.pm
@@ -368,7 +368,11 @@ sub upgrade_data {
     #fix whitespace - before cust_main
     'cust_location' => [],
 
-    #cust_main (remove paycvv from history, locations, cust_payby, etc)
+    # need before cust_main tokenization upgrade,
+    # blocks tokenization upgrade if deprecated features still in use
+    'agent_payment_gateway' => [],
+
+    #cust_main (tokenizes cards, remove paycvv from history, locations, cust_payby, etc)
     'cust_main' => [],
 
     #contact -> cust_contact / prospect_contact
@@ -396,10 +400,6 @@ sub upgrade_data {
     #duplicate history records
     'h_cust_svc'  => [],
 
-    # need before transaction tables, 
-    # blocks tokenization upgrade if deprecated features still in use
-    'agent_payment_gateway' => [],
-
     #populate cust_pay.otaker
     'cust_pay'    => [],
 
diff --git a/FS/FS/agent.pm b/FS/FS/agent.pm
index 8aa78c2..b97e9b9 100644
--- a/FS/FS/agent.pm
+++ b/FS/FS/agent.pm
@@ -9,6 +9,7 @@ use FS::cust_main;
 use FS::cust_pkg;
 use FS::reg_code;
 use FS::agent_payment_gateway;
+use FS::payment_gateway;
 use FS::TicketSystem;
 use FS::Conf;
 
@@ -253,12 +254,7 @@ the business-onlinepayment-ach gateway will be returned if available.
 If I<thirdparty> is set and the I<method> is PAYPAL, the defined paypal
 gateway will be returned.
 
-If I<load_gatewaynum> exists, then either the specified gateway or the
-default gateway will be returned.  Agent overrides are ignored, and this can
-safely be called as a class method if this option is specified.  Not
-compatible with I<thirdparty>.
-
-Exsisting I<$conf> may be passed for efficiency.
+Exisisting I<$conf> may be passed for efficiency.
 
 =cut
 
@@ -268,8 +264,8 @@ Exsisting I<$conf> may be passed for efficiency.
 sub payment_gateway {
   my ( $self, %options ) = @_;
   
+  $options{'conf'} ||= new FS::Conf;
   my $conf = $options{'conf'};
-  $conf ||= new FS::Conf;
 
   if ( $options{thirdparty} ) {
 
@@ -299,72 +295,12 @@ sub payment_gateway {
     }
   }
 
-  my ($override, $payment_gateway);
-  if (exists $options{'load_gatewaynum'}) { # no agent overrides if this opt is in use
-    if ($options{'load_gatewaynum'}) {
-      $payment_gateway = qsearchs('payment_gateway', { gatewaynumnum => $options{'load_gatewaynum'} } );
-      # always fatal
-      die "Could not load payment gateway ".$options{'load_gatewaynum'} unless $payment_gateway;
-    } # else use default, loaded below
-  } else {
-    $override = qsearchs('agent_payment_gateway', { agentnum => $self->agentnum } );
-  }
-
-  if ( $override ) { #use a payment gateway override
-
-    $payment_gateway = $override->payment_gateway;
-
-    $payment_gateway->gateway_namespace('Business::OnlinePayment')
-      unless $payment_gateway->gateway_namespace;
-
-  } elsif (!$payment_gateway) { #use the standard settings from the config
-
-    # the standard settings from the config could be moved to a null agent
-    # agent_payment_gateway referenced payment_gateway
-
-    # remember, this block might be run as a class method if false load_gatewaynum exists
+  my $override = qsearchs('agent_payment_gateway', { agentnum => $self->agentnum } );
 
-    unless ( $conf->exists('business-onlinepayment') ) {
-      if ( $options{'nofatal'} ) {
-        return '';
-      } else {
-        die "Real-time processing not enabled\n";
-      }
-    }
-
-    #load up config
-    my $bop_config = 'business-onlinepayment';
-    $bop_config .= '-ach'
-      if ( $options{method}
-           && $options{method} =~ /^(ECHECK|CHEK)$/
-           && $conf->exists($bop_config. '-ach')
-         );
-    my ( $processor, $login, $password, $action, @bop_options ) =
-      $conf->config($bop_config);
-    $action ||= 'normal authorization';
-    pop @bop_options if scalar(@bop_options) % 2 && $bop_options[-1] =~ /^\s*$/;
-    die "No real-time processor is enabled - ".
-        "did you set the business-onlinepayment configuration value?\n"
-      unless $processor;
-
-    $payment_gateway = new FS::payment_gateway;
-
-    $payment_gateway->gateway_namespace( $conf->config('business-onlinepayment-namespace') ||
-                                 'Business::OnlinePayment');
-    $payment_gateway->gateway_module($processor);
-    $payment_gateway->gateway_username($login);
-    $payment_gateway->gateway_password($password);
-    $payment_gateway->gateway_action($action);
-    $payment_gateway->set('options', [ @bop_options ]);
-
-  }
-
-  unless ( $payment_gateway->gateway_namespace ) {
-    $payment_gateway->gateway_namespace(
-      scalar($conf->config('business-onlinepayment-namespace'))
-      || 'Business::OnlinePayment'
-    );
-  }
+  my $payment_gateway = FS::payment_gateway->by_key_or_default(
+    gatewaynum => $override ? $override->gatewaynum : '',
+    %options,
+  );
 
   $payment_gateway;
 }
diff --git a/FS/FS/cust_main.pm b/FS/FS/cust_main.pm
index a2c0ee8..71552b0 100644
--- a/FS/FS/cust_main.pm
+++ b/FS/FS/cust_main.pm
@@ -5786,6 +5786,11 @@ sub _upgrade_data { #class method
 
 }
 
+sub queueable_upgrade {
+  my $class = shift;
+  FS::cust_main::Billing_Realtime::token_check(@_);
+}
+
 =back
 
 =head1 BUGS
diff --git a/FS/FS/cust_main/Billing_Realtime.pm b/FS/FS/cust_main/Billing_Realtime.pm
index 183a7e6..fb0c010 100644
--- a/FS/FS/cust_main/Billing_Realtime.pm
+++ b/FS/FS/cust_main/Billing_Realtime.pm
@@ -14,6 +14,7 @@ use FS::cust_pay_pending;
 use FS::cust_bill_pay;
 use FS::cust_refund;
 use FS::banned_pay;
+use FS::payment_gateway;
 
 $realtime_bop_decline_quiet = 0;
 
@@ -2296,7 +2297,7 @@ sub realtime_tokenize {
     'type'           => 'CC',
     _bop_auth(\%options),          
     'action'         => 'Tokenize',
-    'description'    => $options{'description'}
+    'description'    => $options{'description'},
     %$bop_content,
     %content, #after
   );
@@ -2346,7 +2347,7 @@ sub tokenized {
   FS::cust_pay->tokenized($payinfo);
 }
 
-=item token_check
+=item token_check [ quiet => 1, queue => 1, daily => 1 ]
 
 NOT A METHOD.  Acts on all customers.  Placed here because it makes
 use of module-internal methods, and to keep everything that uses
@@ -2355,74 +2356,138 @@ Billing::OnlinePayment all in one place.
 Tokenizes all tokenizable card numbers from payinfo in cust_payby and 
 CARD transactions in cust_pay_pending, cust_pay, cust_pay_void and cust_refund.
 
-If all configured gateways have the ability to tokenize, then detection of
-an untokenizable record will cause a fatal error.
+If the I<queue> flag is set, newly tokenized records will be immediately
+committed, regardless of AutoCommit, so as to release the mutex on the record.
+
+If all configured gateways have the ability to tokenize, detection of an 
+untokenizable record will cause a fatal error.  However, if the I<queue> flag 
+is set, this will instead cause a critical error to be recorded in the log, 
+and any other tokenizable records will still be committed.
+
+If the I<daily> flag is also set, detection of existing untokenized records will 
+record a critical error in the system log (because they should have never appeared 
+in the first place.)  Tokenization will still be attempted.
+
+If any configured gateways do NOT have the ability to tokenize, or if a
+default gateway is not configured, then untokenized records are not considered 
+a threat, and no critical errors will be generated in the log.
 
 =cut
 
 sub token_check {
-  # no input, acts on all customers
+  #acts on all customers
+  my %opt = @_;
+  my $debug = !$opt{'quiet'} || $DEBUG;
 
-  eval "use FS::Cursor";  
-  return "Error initializing FS::Cursor: ".$@ if $@;
+  warn "token_check called with opts\n".Dumper(\%opt) if $debug;
 
-  my $dbh = dbh;
+  # force some explicitness when invoking this method
+  die "token_check must run with queue flag if run with daily flag"
+    if $opt{'daily'} && !$opt{'queue'};
+
+  my $conf = FS::Conf->new;
+
+  my $log = FS::Log->new('FS::cust_main::Billing_Realtime::token_check');
 
-  # get list of all gateways in table (not counting default gateway)
   my $cache = {}; #cache for module info
-  my $sth = $dbh->prepare('SELECT DISTINCT gatewaynum FROM payment_gateway')
-    or die $dbh->errstr;
-  $sth->execute or die $sth->errstr;
-  my @gatewaynums;
-  while (my $row = $sth->fetchrow_hashref) {
-    push(@gatewaynums,$row->{'gatewaynum'});
-  }
-  $sth->finish;
 
   # look for a gateway that can't tokenize
-  my $disallow_untokenized = 1;
-  foreach my $gatewaynum ('',@gatewaynums) {
-    my $gateway = FS::agent->payment_gateway( load_gatewaynum => $gatewaynum, nofatal => 1 );
-    if (!$gateway) { # already died if $gatewaynum
+  my $require_tokenized = 1;
+  foreach my $gateway (
+    FS::payment_gateway->all_gateways(
+      'method'  => 'CC',
+      'conf'    => $conf,
+      'nofatal' => 1,
+    )
+  ) {
+    if (!$gateway) {
       # no default gateway, no promise to tokenize
       # can just load other gateways as-needeed below
-      $disallow_untokenized = 0;
+      $require_tokenized = 0;
       last;
     }
     my $info = _token_check_gateway_info($cache,$gateway);
-    return $info unless ref($info); # means it's an error message
+    die $info unless ref($info); # means it's an error message
     unless ($info->{'can_tokenize'}) {
       # a configured gateway can't tokenize, that's all we need to know right now
       # can just load other gateways as-needeed below
-      $disallow_untokenized = 0;
+      $require_tokenized = 0;
       last;
     }
   }
 
+  warn "REQUIRE TOKENIZED" if $require_tokenized && $debug;
+
+  # upgrade does not call this with autocommit turned on,
+  # and autocommit will be ignored if opt queue is set,
+  # but might as well be thorough...
   my $oldAutoCommit = $FS::UID::AutoCommit;
   local $FS::UID::AutoCommit = 0;
+  my $dbh = dbh;
+
+  # for retrieving data in chunks
+  my $step = 500;
+  my $offset = 0;
 
   ### Tokenize cust_payby
 
-  my $cust_search = FS::Cursor->new({ table => 'cust_main' },$dbh);
-  while (my $cust_main = $cust_search->fetch) {
+  my @recnums;
+
+CUSTLOOP:
+  while (my $custnum = _token_check_next_recnum($dbh,'cust_main',$step,\$offset,\@recnums)) {
+    my $cust_main = FS::cust_main->by_key($custnum);
+    my $payment_gateway;
     foreach my $cust_payby ($cust_main->cust_payby('CARD','DCRD')) {
-      next if $cust_payby->tokenized;
-      # load gateway first, just so we can cache it
-      my $payment_gateway = $cust_main->_payment_gateway({
-        'nofatal' => 1, # handle error smoothly below
+
+      # see if it's already tokenized
+      if ($cust_payby->tokenized) {
+        warn "cust_payby ".$cust_payby->get($cust_payby->primary_key)." already tokenized" if $debug;
+        next;
+      }
+
+      if ($require_tokenized && $opt{'daily'}) {
+        $log->critical("Untokenized card number detected in cust_payby ".$cust_payby->custpaybynum);
+        $dbh->commit or die $dbh->errstr; # commit log message
+      }
+
+      # only load gateway if we need to, and only need to load it once
+      my $payment_gateway ||= $cust_main->_payment_gateway({
+        'method'  => 'CC',
+        'conf'    => $conf,
+        'nofatal' => 1, # handle lack of gateway smoothly below
       });
       unless ($payment_gateway) {
         # no reason to have untokenized card numbers saved if no gateway,
-        #   but only fatal if we expected everyone to tokenize card numbers
-        next unless $disallow_untokenized;
-        $cust_search->DESTROY;
+        #   but only a problem if we expected everyone to tokenize card numbers
+        unless ($require_tokenized) {
+          warn "Skipping cust_payby for cust_main ".$cust_main->custnum.", no payment gateway" if $debug;
+          next CUSTLOOP; # can skip rest of customer
+        }
+        my $error = "No gateway found for custnum ".$cust_main->custnum;
+        if ($opt{'queue'}) {
+          $log->critical($error);
+          $dbh->commit or die $dbh->errstr; # commit error message
+          next; # not next CUSTLOOP, want to record error for every cust_payby
+        }
         $dbh->rollback if $oldAutoCommit;
-        return "No gateway found for custnum ".$cust_main->custnum;
+        die $error;
       }
+
       my $info = _token_check_gateway_info($cache,$payment_gateway);
+      unless (ref($info)) {
+        # only throws error if Business::OnlinePayment won't load,
+        #   which is just cause to abort this whole process, even if queue
+        $dbh->rollback if $oldAutoCommit;
+        die $info; # error message
+      }
       # no fail here--a configured gateway can't tokenize, so be it
-      next unless ref($info) && $info->{'can_tokenize'};
+      unless ($info->{'can_tokenize'}) {
+        warn "Skipping ".$cust_main->custnum." cannot tokenize" if $debug;
+        next;
+      }
+
+      # time to tokenize
+      $cust_payby = $cust_payby->select_for_update;
       my %tokenopts = (
         'payment_gateway' => $payment_gateway,
         'cust_payby'      => $cust_payby,
@@ -2434,11 +2499,20 @@ sub token_check {
         $error ||= 'Unknown error';
       }
       if ($error) {
-        $cust_search->DESTROY;
+        $error = "Error tokenizing cust_payby ".$cust_payby->custpaybynum.": ".$error;
+        if ($opt{'queue'}) {
+          $log->critical($error);
+          $dbh->commit or die $dbh->errstr; # commit log message, release mutex
+          next; # not next CUSTLOOP, want to record error for every cust_payby
+        }
         $dbh->rollback if $oldAutoCommit;
-        return "Error tokenizing cust_payby ".$cust_payby->custpaybynum.": ".$error;
+        die $error;
       }
+      $dbh->commit or die $dbh->errstr if $opt{'queue'}; # release mutex
+      warn "TOKENIZED cust_payby ".$cust_payby->get($cust_payby->primary_key) if $debug;
     }
+    warn "cust_payby upgraded for custnum ".$cust_main->custnum if $debug;
+
   }
 
   ### Tokenize/mask transaction tables
@@ -2449,50 +2523,83 @@ sub token_check {
   # grep assistance:
   #   $cust_pay_pending->replace, $cust_pay->replace, $cust_pay_void->replace, $cust_refund->replace all run here
   foreach my $table ( qw(cust_pay_pending cust_pay cust_pay_void cust_refund) ) {
-    my $search = FS::Cursor->new({
-      table     => $table,
-      hashref   => { 'payby' => 'CARD' },
-    },$dbh);
-    while (my $record = $search->fetch) {
-      next if $record->tokenized;
-      next if !$record->payinfo; #shouldn't happen, but at least it's not a card number
-      next if $record->payinfo =~ /N\/A/; # ??? Not sure why we do this, but it's not a card number
+    warn "Checking $table" if $debug;
+
+    # FS::Cursor does not seem to work over multiple commits (gives cursor not found errors)
+    # loading only record ids, then loading individual records one at a time
+    my $tclass = 'FS::'.$table;
+    $offset = 0;
+    @recnums = ();
+
+    while (my $recnum = _token_check_next_recnum($dbh,$table,$step,\$offset,\@recnums)) {
+      my $record = $tclass->by_key($recnum);
+      if (FS::cust_main::Billing_Realtime->tokenized($record->payinfo)) {
+        warn "Skipping tokenized record for $table ".$record->get($record->primary_key) if $debug;
+        next;
+      }
+      if (!$record->payinfo) { #shouldn't happen, but at least it's not a card number
+        warn "Skipping blank payinfo for $table ".$record->get($record->primary_key) if $debug;
+        next;
+      }
+      if ($record->payinfo =~ /N\/A/) { # ??? Not sure why we do this, but it's not a card number
+        warn "Skipping NA payinfo for $table ".$record->get($record->primary_key) if $debug;
+        next;
+      }
+
+      if ($require_tokenized && $opt{'daily'}) {
+        $log->critical("Untokenized card number detected in $table ".$record->get($record->primary_key));
+        $dbh->commit or die $dbh->errstr; # commit log message
+      }
 
       # don't use customer agent gateway here, use the gatewaynum specified by the record
-      my $gatewaynum = $record->gatewaynum || '';
-      my $gateway = FS::agent->payment_gateway( load_gatewaynum => $gatewaynum );
-      unless ($gateway) { # already died if $gatewaynum
-        # only fatal if we expected everyone to tokenize
-        next unless $disallow_untokenized;
-        $search->DESTROY;
-        $dbh->rollback if $oldAutoCommit;
-        return "No gateway found for $table ".$record->get($record->primary_key);
+      my $gateway = FS::payment_gateway->by_key_or_default( 
+        'method'     => 'CC',
+        'conf'       => $conf,
+        'nofatal'    => 1,
+        'gatewaynum' => $record->gatewaynum || '',
+      );
+      unless ($gateway) {
+        # means no default gateway, no promise to tokenize, can skip
+        warn "Skipping missing gateway for $table ".$record->get($record->primary_key) if $debug;
+        next;
       }
+
       my $info = _token_check_gateway_info($cache,$gateway);
       unless (ref($info)) {
         # only throws error if Business::OnlinePayment won't load,
-        #   which is just cause to abort this whole process
-        $search->DESTROY;
+        #   which is just cause to abort this whole process, even if queue
         $dbh->rollback if $oldAutoCommit;
-        return $info; # error message
+        die $info; # error message
       }
 
       # a configured gateway can't tokenize, move along
-      next unless $info->{'can_tokenize'};
+      unless ($info->{'can_tokenize'}) {
+        warn "Skipping, cannot tokenize $table ".$record->get($record->primary_key) if $debug;
+        next;
+      }
 
       my $cust_main = $record->cust_main;
-      unless ($cust_main || (
+      if (!$cust_main) {
         # might happen for cust_pay_pending from failed verify records,
         #   in which case we attempt tokenization without cust_main
         # everything else should absolutely have a cust_main
-        $table eq 'cust_pay_pending'
-          && $record->{'custnum_pending'}
-          && !$disallow_untokenized
-      )) {
-        $search->DESTROY;
-        $dbh->rollback if $oldAutoCommit;
-        return "Could not load cust_main for $table ".$record->get($record->primary_key);
+        if ($table eq 'cust_pay_pending' && $record->{'custnum_pending'}) {
+          warn "ATTEMPTING GATEWAY-ONLY TOKENIZE" if $debug;
+        } else {
+          my $error = "Could not load cust_main for $table ".$record->get($record->primary_key);
+          if ($opt{'queue'}) {
+            $log->critical($error);
+            $dbh->commit or die $dbh->errstr; # commit log message
+            next;
+          }
+          $dbh->rollback if $oldAutoCommit;
+          die $error;
+        }
       }
+
+      # if we got this far, time to mutex
+      $record = $record->select_for_update;
+
       # no clear record of name/address/etc used for transaction,
       # but will load name/phone/id from customer if run as an object method,
       # so we try that if we can
@@ -2512,19 +2619,44 @@ sub token_check {
         $error ||= 'Unknown error';
       }
       if ($error) {
-        $search->DESTROY;
+        $error = "Error tokenizing $table ".$record->get($record->primary_key).": ".$error;
+        if ($opt{'queue'}) {
+          $log->critical($error);
+          $dbh->commit or die $dbh->errstr; # commit log message, release mutex
+          next;
+        }
         $dbh->rollback if $oldAutoCommit;
-        return "Error tokenizing $table ".$record->get($record->primary_key).": ".$error;
+        die $error;
       }
+      $dbh->commit or die $dbh->errstr if $opt{'queue'}; # release mutex
+      warn "TOKENIZED $table ".$record->get($record->primary_key) if $debug;
+
     } # end record loop
   } # end table loop
 
-  $dbh->commit if $oldAutoCommit;
+  $dbh->commit or die $dbh->errstr if $oldAutoCommit;
 
   return '';
 }
 
 # not a method!
+sub _token_check_next_recnum {
+  my ($dbh,$table,$step,$offset,$recnums) = @_;
+  my $recnum = shift @$recnums;
+  return $recnum if $recnum;
+  my $tclass = 'FS::'.$table;
+  my $sth = $dbh->prepare('SELECT '.$tclass->primary_key.' FROM '.$table.' ORDER BY '.$tclass->primary_key.' LIMIT '.$step.' OFFSET '.$$offset) or die $dbh->errstr;
+  $sth->execute() or die $sth->errstr;
+  my @recnums;
+  while (my $rec = $sth->fetchrow_hashref) {
+    push @$recnums, $rec->{$tclass->primary_key};
+  }
+  $sth->finish();
+  $$offset += $step;
+  return shift @$recnums;
+}
+
+# not a method!
 sub _token_check_gateway_info {
   my ($cache,$payment_gateway) = @_;
 
@@ -2562,8 +2694,6 @@ sub _token_check_gateway_info {
   $info->{'void_requires_card'} = 1
     if $transaction->info('CC_void_requires_card');
 
-  $cache->{$payment_gateway->gateway_module} = $info;
-
   return $info;
 }
 
diff --git a/FS/FS/log_context.pm b/FS/FS/log_context.pm
index 51aa79d..a41d3c8 100644
--- a/FS/FS/log_context.pm
+++ b/FS/FS/log_context.pm
@@ -11,6 +11,7 @@ my @contexts = ( qw(
   FS::cust_main::Billing_Realtime::realtime_bop
   FS::cust_main::Billing_Realtime::realtime_tokenize
   FS::cust_main::Billing_Realtime::realtime_verify_bop
+  FS::cust_main::Billing_Realtime::token_check
   FS::pay_batch::import_from_gateway
   FS::part_pkg
   FS::Misc::Geo::standardize_uscensus
diff --git a/FS/FS/payinfo_Mixin.pm b/FS/FS/payinfo_Mixin.pm
index 2f50312..be37568 100644
--- a/FS/FS/payinfo_Mixin.pm
+++ b/FS/FS/payinfo_Mixin.pm
@@ -468,6 +468,7 @@ Optionally, an arbitrary payby and payinfo can be passed.
 sub tokenized {
   my $self = shift;
   my $payinfo = scalar(@_) ? shift : $self->payinfo;
+  return 0 unless $payinfo; #avoid uninitialized value error
   $payinfo =~ /^99\d{14}$/;
 }
 
diff --git a/FS/FS/payment_gateway.pm b/FS/FS/payment_gateway.pm
index afae266..170d37a 100644
--- a/FS/FS/payment_gateway.pm
+++ b/FS/FS/payment_gateway.pm
@@ -323,6 +323,108 @@ sub processor {
   }
 }
 
+=item default_gateway OPTIONS
+
+Class method.
+
+Returns default gateway (from business-onlinepayment conf) as a payment_gateway object.
+
+Accepts options
+
+conf - existing conf object
+
+nofatal - return blank instead of dying if no default gateway is configured
+
+method - if set to CHEK or ECHECK, returns object for business-onlinepayment-ach if available
+
+Before using this, be sure you wouldn't rather be using L</by_key_or_default> or,
+more likely, L<FS::agent/payment_gateway>.
+
+=cut
+
+# the standard settings from the config could be moved to a null agent
+# agent_payment_gateway referenced payment_gateway
+
+sub default_gateway {
+  my ($self,%options) = @_;
+
+  $options{'conf'} ||= new FS::Conf;
+  my $conf = $options{'conf'};
+
+  unless ( $conf->exists('business-onlinepayment') ) {
+    if ( $options{'nofatal'} ) {
+      return '';
+    } else {
+      die "Real-time processing not enabled\n";
+    }
+  }
+
+  #load up config
+  my $bop_config = 'business-onlinepayment';
+  $bop_config .= '-ach'
+    if ( $options{method}
+         && $options{method} =~ /^(ECHECK|CHEK)$/
+         && $conf->exists($bop_config. '-ach')
+       );
+  my ( $processor, $login, $password, $action, @bop_options ) =
+    $conf->config($bop_config);
+  $action ||= 'normal authorization';
+  pop @bop_options if scalar(@bop_options) % 2 && $bop_options[-1] =~ /^\s*$/;
+  die "No real-time processor is enabled - ".
+      "did you set the business-onlinepayment configuration value?\n"
+    unless $processor;
+
+  my $payment_gateway = new FS::payment_gateway;
+  $payment_gateway->gateway_namespace( $conf->config('business-onlinepayment-namespace') ||
+                                       'Business::OnlinePayment');
+  $payment_gateway->gateway_module($processor);
+  $payment_gateway->gateway_username($login);
+  $payment_gateway->gateway_password($password);
+  $payment_gateway->gateway_action($action);
+  $payment_gateway->set('options', [ @bop_options ]);
+  return $payment_gateway;
+}
+
+=item by_key_or_default OPTIONS
+
+Either returns the gateway specified by option gatewaynum, or the default gateway.
+
+Accepts the same options as L</default_gateway>.
+
+Also ensures that the gateway_namespace has been set.
+
+=cut
+
+sub by_key_or_default {
+  my ($self,%options) = @_;
+
+  if ($options{'gatewaynum'}) {
+    my $payment_gateway = $self->by_key($options{'gatewaynum'});
+    # regardless of nofatal, which is only meant for handling lack of default gateway
+    die "payment_gateway ".$options{'gatewaynum'}." not found"
+      unless $payment_gateway;
+    $payment_gateway->gateway_namespace('Business::OnlinePayment')
+      unless $payment_gateway->gateway_namespace;
+    return $payment_gateway;
+  } else {
+    return $self->default_gateway(%options);
+  }
+}
+
+# if it weren't for the way gateway_namespace default is set, this method would not be necessary
+# that should really go in check() with an accompanying upgrade, so we could just use qsearch safely,
+# but currently short on time to test deeper changes...
+#
+# if no default gateway is set and nofatal is passed, first value returned is blank string
+sub all_gateways {
+  my ($self,%options) = @_;
+  my @out;
+  foreach my $gatewaynum ('',( map {$_->gatewaynum} qsearch('payment_gateway') )) {
+    push @out, $self->by_key_or_default( %options, gatewaynum => $gatewaynum );
+  }
+  return @out;
+}
+
 # _upgrade_data
 #
 # Used by FS::Upgrade to migrate to a new database.
diff --git a/FS/bin/freeside-daily b/FS/bin/freeside-daily
index ee95c14..e1463f5 100755
--- a/FS/bin/freeside-daily
+++ b/FS/bin/freeside-daily
@@ -97,7 +97,7 @@ use FS::Cron::backup qw(backup);
 backup();
 
 #except we'd rather not start cleanup jobs until the backup is done
-cleanup();
+cleanup( quiet => !$opt{'v'} );
 
 $log->info('finish');
 
diff --git a/FS/t/suite/13-tokenization.t b/FS/t/suite/13-tokenization.t
new file mode 100755
index 0000000..1b654ad
--- /dev/null
+++ b/FS/t/suite/13-tokenization.t
@@ -0,0 +1,82 @@
+#!/usr/bin/perl
+
+use FS::Test;
+use Test::More tests => 8;
+use FS::Conf;
+
+### can only run on test database (company name "Freeside Test")
+### will run upgrade, which uses lots of prints & warns beyond regular test output
+
+my $fs = FS::Test->new( user => 'admin' );
+my $conf = new_ok('FS::Conf');
+my $err;
+my $bopconf;
+
+like( $conf->config('company_name'), qr/^Freeside Test/, 'using test database' ) or BAIL_OUT('');
+
+# some pre-upgrade cleanup, upgrade will fail if these are still configured
+foreach my $cust_main ( $fs->qsearch('cust_main') ) {
+  my @count = $fs->qsearch('agent_payment_gateway', { agentnum => $cust_main->agentnum } );
+  if (@count > 1) {
+    note("DELETING CARDTYPE GATEWAYS");
+    foreach my $apg (@count) {
+      $err = $apg->delete if $apg->cardtype;
+      last if $err;
+    }
+    @count = $fs->qsearch('agent_payment_gateway', { agentnum => $cust_main->agentnum } );
+    if (@count > 1) {
+      $err = "Still found ".@count." gateways for custnum ".$cust_main->custnum;
+      last;
+    }
+  }
+}
+ok( !$err, "remove obsolete payment gateways" ) or BAIL_OUT($err);
+
+$bopconf = 
+'IPPay
+TESTTERMINAL';
+$conf->set('business-onlinepayment' => $bopconf);
+is( join("\n",$conf->config('business-onlinepayment')), $bopconf, "setting first default gateway" ) or BAIL_OUT('');
+
+$err = system('freeside-upgrade','admin');
+ok( !$err, 'initial upgrade' ) or BAIL_OUT('Error string: '.$!);
+
+$bopconf =
+'CardFortress
+cardfortresstest
+(TEST54)
+Normal Authorization
+gateway
+IPPay
+gateway_login
+TESTTERMINAL
+gateway_password
+
+private_key
+/usr/local/etc/freeside/cardfortresstest.txt';
+$conf->set('business-onlinepayment' => $bopconf);
+is( join("\n",$conf->config('business-onlinepayment')), $bopconf, "setting tokenizable default gateway" ) or BAIL_OUT('');
+
+foreach my $pg ($fs->qsearch('payment_gateway')) {
+  unless ($pg->gateway_module eq 'CardFortress') {
+    note('UPGRADING NON-CF PAYMENT GATEWAY');
+    my %pgopts = (
+      gateway          => $pg->gateway_module,
+      gateway_login    => $pg->gateway_username,
+      gateway_password => $pg->gateway_password,
+      private_key      => '/usr/local/etc/freeside/cardfortresstest.txt',
+    );
+    $pg->gateway_module('CardFortress');
+    $pg->gateway_username('cardfortresstest');
+    $pg->gateway_password('(TEST54)');
+    $err = $pg->replace(\%pgopts);
+    last if $err;
+  }
+}
+ok( !$err, "remove non-CF payment gateways" ) or BAIL_OUT($err);
+
+$err = system('freeside-upgrade','admin');
+ok( !$err, 'tokenizable upgrade' ) or BAIL_OUT('Error string: '.$!);
+
+1;
+
diff --git a/httemplate/edit/elements/edit.html b/httemplate/edit/elements/edit.html
index 35818dd..7e0eee4 100644
--- a/httemplate/edit/elements/edit.html
+++ b/httemplate/edit/elements/edit.html
@@ -247,7 +247,7 @@ Example:
   >
 
   <INPUT TYPE="hidden" NAME="svcdb" VALUE="<% $table %>">
-  <INPUT TYPE="hidden" NAME="<% $pkey %>" VALUE="<% $clone ? '' : $object->$pkey() %>">
+  <INPUT TYPE="hidden" ID="<% $pkey %>" NAME="<% $pkey %>" VALUE="<% $clone ? '' : $object->$pkey() %>">
 
   <% defined($opt{'form_init'}) 
         ? ( ref($opt{'form_init'})
diff --git a/httemplate/edit/payment_gateway.html b/httemplate/edit/payment_gateway.html
index b44b315..f9b8f24 100644
--- a/httemplate/edit/payment_gateway.html
+++ b/httemplate/edit/payment_gateway.html
@@ -22,6 +22,9 @@
 <SCRIPT TYPE="text/javascript">
   var modulesForNamespace = <% $json->encode(\%modules) %>;
   function changeNamespace() {
+    if (document.getElementById('gatewaynum').value) {
+      return true;
+    }
     var ns = document.getElementById('gateway_namespace').value;
     var select_module = document.getElementById('gateway_module');
     select_module.options.length = 0;
@@ -180,7 +183,13 @@ my $field_callback = sub {
   my ($cgi, $object, $field_hashref ) = @_;
   if ($object->gatewaynum) {
     if ( $field_hashref->{field} eq 'gateway_module' ) {
-      $field_hashref->{type} = 'fixed';
+      if ($object->gateway_namespace eq 'Business::OnlinePayment' &&
+          $object->gateway_module ne 'CardFortress'
+      ) {
+        $field_hashref->{options} = [ $object->gateway_module, 'CardFortress' ]
+      } else {
+        $field_hashref->{type} = 'fixed';
+      }
     } elsif ( $field_hashref->{field} eq 'gateway_namespace' ) {
       $field_hashref->{type} = 'fixed';
       $field_hashref->{formatted_value} = $object->namespace_description;