faster (cached) fuzzy searches

prelim. job queues!
fixed part_svc editing

22 files changed, 770 insertions, 38 deletions

diff --git a/FS/FS.pm b/FS/FS.pm
index cd11e96..d2f47d7 100644
--- a/FS/FS.pm
+++ b/FS/FS.pm
@@ -93,7 +93,7 @@ L<FS::cust_refund> - Refund class
 
 L<FS::cust_credit_refund> - Refund application class
 
-L<FS::cust_credit_bill> - Refund invoice application class
+L<FS::cust_credit_bill> - Credit invoice application class
 
 L<FS::cust_pay_batch> - Credit card transaction queue class
 
@@ -105,6 +105,10 @@ L<FS::port> - NAS port class
 
 L<FS::session> - User login session class
 
+L<FS::queue> - Job queue
+
+L<FS::queue_arg> - Job arguments
+
 =head2 User Interface classes (under development; not yet usable)
 
 L<FS::UI::Base> - User-interface base class
@@ -135,7 +139,7 @@ The main documentation is in htdocs/docs.
 
 =head1 VERSION
 
-$Id: FS.pm,v 1.7 2001-09-06 20:41:59 ivan Exp $
+$Id: FS.pm,v 1.8 2001-09-11 00:08:18 ivan Exp $
 
 =head1 SUPPORT
 
diff --git a/FS/FS/Record.pm b/FS/FS/Record.pm
index 9a532e3..b476df2 100644
--- a/FS/FS/Record.pm
+++ b/FS/FS/Record.pm
@@ -855,13 +855,30 @@ Untaints arbitrary data.  Be careful.
 =cut
 
 sub ut_anything {
-  my($self,$field)=@_;
+  my( $self, $field ) = @_;
   $self->getfield($field) =~ /^(.*)$/s
     or return "Illegal $field: ". $self->getfield($field);
   $self->setfield($field,$1);
   '';
 }
 
+=item ut_enum COLUMN CHOICES_ARRAYREF
+
+Check/untaint a column, supplying all possible choices, like the "enum" type.
+
+=cut
+
+sub ut_enum {
+  my( $self, $field, $choices ) = @_;
+  foreach my $choice ( @$choices ) {
+    if ( $self->getfield($field) eq $choice ) {
+      $self->setfield($choice);
+      return '';
+    }
+  }
+  return "Illegal (enum) field $field: ". $self->getfield($field);
+}
+
 =item fields [ TABLE ]
 
 This can be used as both a subroutine and a method call.  It returns a list
@@ -976,7 +993,7 @@ sub DESTROY { return; }
 
 =head1 VERSION
 
-$Id: Record.pm,v 1.26 2001-08-31 09:20:35 ivan Exp $
+$Id: Record.pm,v 1.27 2001-09-11 00:08:18 ivan Exp $
 
 =head1 BUGS
 
diff --git a/FS/FS/cust_credit_bill.pm b/FS/FS/cust_credit_bill.pm
index 58cc789..9613408 100644
--- a/FS/FS/cust_credit_bill.pm
+++ b/FS/FS/cust_credit_bill.pm
@@ -33,12 +33,14 @@ FS::cust_credit_bill - Object methods for cust_credit_bill records
 =head1 DESCRIPTION
 
 An FS::cust_credit_bill object represents application of a credit (see
-L<FS::cust_credit>) to a customer bill (see L<FS::cust_bill>).  FS::cust_credit
+L<FS::cust_credit>) to an invoice (see L<FS::cust_bill>).  FS::cust_credit
 inherits from FS::Record.  The following fields are currently supported:
 
 =over 4
 
-=item crednum - primary key; credit being applied 
+=item creditbillnum - primary key
+
+=item crednum - credit being applied 
 
 =item invnum - invoice to which credit is applied (see L<FS::cust_bill>)
 
@@ -119,7 +121,7 @@ sub check {
 
   $self->_date(time) unless $self->_date;
 
-  return "Cannot apply more than remaining value of credit memo"
+  return "Cannot apply more than remaining value of credit"
     unless $self->amount <= $cust_credit->credited;
 
   return "Cannot apply more than remaining value of invoice"
@@ -143,7 +145,7 @@ sub cust_credit {
 
 =head1 VERSION
 
-$Id: cust_credit_bill.pm,v 1.4 2001-09-02 07:49:52 ivan Exp $
+$Id: cust_credit_bill.pm,v 1.5 2001-09-11 00:08:18 ivan Exp $
 
 =head1 BUGS
 
diff --git a/FS/FS/cust_main.pm b/FS/FS/cust_main.pm
index b29e385..e4f55cf 100644
--- a/FS/FS/cust_main.pm
+++ b/FS/FS/cust_main.pm
@@ -31,6 +31,7 @@ use FS::cust_main_invoice;
 use FS::cust_credit_bill;
 use FS::cust_bill_pay;
 use FS::prepay_credit;
+use FS::queue;
 
 @ISA = qw( FS::Record );
 
@@ -343,6 +344,22 @@ sub insert {
     }
   }
 
+  my $queue = new FS::queue { 'job' => 'FS::cust_main::append_fuzzyfiles' };
+  $error = $queue->insert($self->getfield('last'), $self->company);
+  if ( $error ) {
+    $dbh->rollback if $oldAutoCommit;
+    return "queueing job (transaction rolled back): $error";
+  }
+
+  if ( defined $self->dbdef_table->column('ship_last') && $self->ship_last ) {
+    $queue = new FS::queue { 'job' => 'FS::cust_main::append_fuzzyfiles' };
+    $error = $queue->insert($self->getfield('last'), $self->company);
+    if ( $error ) {
+      $dbh->rollback if $oldAutoCommit;
+      return "queueing job (transaction rolled back): $error";
+    }
+  }
+
   $dbh->commit or die $dbh->errstr if $oldAutoCommit;
   '';
 
@@ -1533,24 +1550,140 @@ sub referral_cust_main {
 
 =over 4
 
-=item rebuild_fuzzyfile
+=item check_and_rebuild_fuzzyfiles
+
+=cut
+
+sub check_and_rebuild_fuzzyfiles {
+  my $dir = $FS::UID::conf_dir. "cache.". $FS::UID::datasrc;
+  -e "$dir/cust_main.last" && -e "$dir/cust_main.company"
+    or &rebuild_fuzzyfiles;
+}
+
+=item rebuild_fuzzyfiles
 
 =cut
 
 sub rebuild_fuzzyfiles {
+
+  use Fcntl qw(:flock);
+
+  my $dir = $FS::UID::conf_dir. "cache.". $FS::UID::datasrc;
+
+  #last
+
+  open(LASTLOCK,">>$dir/cust_main.last")
+    or die "can't open $dir/cust_main.last: $!";
+  flock(LASTLOCK,LOCK_EX)
+    or die "can't lock $dir/cust_main.last: $!";
+
   my @all_last = map $_->getfield('last'), qsearch('cust_main', {});
   push @all_last,
                  grep $_, map $_->getfield('ship_last'), qsearch('cust_main',{})
-      if defined dbdef->table('cust_main')->column('ship_last');
-#  open(
+    if defined dbdef->table('cust_main')->column('ship_last');
+
+  open (LASTCACHE,">$dir/cust_main.last.tmp")
+    or die "can't open $dir/cust_main.last.tmp: $!";
+  print LASTCACHE join("\n", @all_last), "\n";
+  close LASTCACHE or die "can't close $dir/cust_main.last.tmp: $!";
+
+  rename "$dir/cust_main.last.tmp", "$dir/cust_main.last";
+  close LASTLOCK;
+
+  #company
+
+  open(COMPANYLOCK,">>$dir/cust_main.company")
+    or die "can't open $dir/cust_main.company: $!";
+  flock(COMPANYLOCK,LOCK_EX)
+    or die "can't lock $dir/cust_main.company: $!";
+
+  my @all_company = grep $_ ne '', map $_->company, qsearch('cust_main',{});
+  push @all_company,
+       grep $_ ne '', map $_->ship_company, qsearch('cust_main', {})
+    if defined dbdef->table('cust_main')->column('ship_last');
+
+  open (COMPANYCACHE,">$dir/cust_main.company.tmp")
+    or die "can't open $dir/cust_main.company.tmp: $!";
+  print COMPANYCACHE join("\n", @all_company), "\n";
+  close COMPANYCACHE or die "can't close $dir/cust_main.company.tmp: $!";
+
+  rename "$dir/cust_main.company.tmp", "$dir/cust_main.company";
+  close COMPANYLOCK;
 
 }
 
-=back
+=item all_last
+
+=cut
+
+sub all_last {
+  my $dir = $FS::UID::conf_dir. "cache.". $FS::UID::datasrc;
+  open(LASTCACHE,"<$dir/cust_main.last")
+    or die "can't open $dir/cust_main.last: $!";
+  my @array = split(/\n/, <LASTCACHE> );
+  close LASTCACHE;
+  \@array;
+}
+
+=item all_company
+
+=cut
+
+sub all_company {
+  my $dir = $FS::UID::conf_dir. "cache.". $FS::UID::datasrc;
+  open(COMPANYCACHE,"<$dir/cust_main.company")
+    or die "can't open $dir/cust_main.last: $!";
+  my @array = split(/\n/, <COMPANYCACHE> );
+  close COMPANYCACHE;
+  \@array;
+}
+
+=item append_fuzzyfiles LASTNAME COMPANY
+
+=cut
+
+sub append_fuzzyfiles {
+  my( $last, $company ) = @_;
+
+  use Fcntl qw(:flock);
+
+  my $dir = $FS::UID::conf_dir. "cache.". $FS::UID::datasrc;
+
+  if ( $last ) {
+
+    open(LAST,">>$dir/cust_main.last")
+      or die "can't open $dir/cust_main.last: $!";
+    flock(LAST,LOCK_EX)
+      or die "can't lock $dir/cust_main.last: $!";
+
+    print LAST "$last\n";
+
+    flock(LAST,LOCK_UN)
+      or die "can't unlock $dir/cust_main.last: $!";
+    close LAST;
+  }
+
+  if ( $company ) {
+
+    open(COMPANY,">>$dir/cust_main.company")
+      or die "can't open $dir/cust_main.company: $!";
+    flock(COMPANY,LOCK_EX)
+      or die "can't lock $dir/cust_main.company: $!";
+
+    print COMPANY "$company\n";
+
+    flock(COMPANY,LOCK_UN)
+      or die "can't unlock $dir/cust_main.company: $!";
+
+    close COMPANY;
+  }
+
+  1;
+}
 
 =head1 VERSION
 
-$Id: cust_main.pm,v 1.29 2001-09-03 22:07:38 ivan Exp $
+$Id: cust_main.pm,v 1.30 2001-09-11 00:08:18 ivan Exp $
 
 =head1 BUGS
 
diff --git a/FS/FS/part_svc.pm b/FS/FS/part_svc.pm
index e64f09a..f1e71ad 100644
--- a/FS/FS/part_svc.pm
+++ b/FS/FS/part_svc.pm
@@ -117,7 +117,7 @@ sub insert {
         $error = $part_svc_column->insert;
       }
     } else {
-      $error = $part_svc_column->delete;
+      $error = $previous ? $previous->delete : '';
     }
     if ( $error ) {
       $dbh->rollback if $oldAutoCommit;
@@ -155,7 +155,64 @@ sub replace {
   return "Can't change svcdb for an existing service definition!"
     unless $old->svcdb eq $new->svcdb;
 
-  $new->SUPER::replace( $old );
+  local $SIG{HUP} = 'IGNORE';
+  local $SIG{INT} = 'IGNORE';
+  local $SIG{QUIT} = 'IGNORE';
+  local $SIG{TERM} = 'IGNORE';
+  local $SIG{TSTP} = 'IGNORE';
+  local $SIG{PIPE} = 'IGNORE';
+
+  my $oldAutoCommit = $FS::UID::AutoCommit;
+  local $FS::UID::AutoCommit = 0;
+  my $dbh = dbh;
+
+  my $error = $new->SUPER::replace( $old );
+  if ( $error ) {
+    $dbh->rollback if $oldAutoCommit;
+    return $error;
+  }
+
+  if ( @_ && $_[0] eq '1.3-COMPAT' ) {
+    my $svcdb = $new->svcdb;
+    foreach my $field (
+      grep { $_ ne 'svcnum'
+             && defined( $new->getfield($svcdb.'__'.$_.'_flag') )
+           } fields($svcdb)
+    ) {
+      my $part_svc_column = $new->part_svc_column($field);
+      my $previous = qsearchs('part_svc_column', {
+        'svcpart'    => $new->svcpart,
+        'columnname' => $field,
+      } );
+
+      my $flag = $new->getfield($svcdb.'__'.$field.'_flag');
+      if ( uc($flag) =~ /^([DF])$/ ) {
+        $part_svc_column->setfield('columnflag', $1);
+        $part_svc_column->setfield('columnvalue',
+          $new->getfield($svcdb.'__'.$field)
+        );
+        if ( $previous ) {
+          $error = $part_svc_column->replace($previous);
+        } else {
+          $error = $part_svc_column->insert;
+        }
+      } else {
+        $error = $previous ? $previous->delete : '';
+      }
+      if ( $error ) {
+        $dbh->rollback if $oldAutoCommit;
+        return $error;
+      }
+    }
+  } else {
+    $dbh->rollback if $oldAutoCommit;
+    return 'non-1.3-COMPAT interface not yet written';
+    #not yet implemented
+  }
+
+  $dbh->commit or die $dbh->errstr if $oldAutoCommit;
+
+  '';
 }
 
 =item check
@@ -232,7 +289,7 @@ sub part_svc_column {
 
 =head1 VERSION
 
-$Id: part_svc.pm,v 1.3 2001-09-06 20:41:59 ivan Exp $
+$Id: part_svc.pm,v 1.4 2001-09-11 00:08:18 ivan Exp $
 
 =head1 BUGS
 
diff --git a/FS/FS/queue.pm b/FS/FS/queue.pm
new file mode 100644
index 0000000..51dd9af
--- /dev/null
+++ b/FS/FS/queue.pm
@@ -0,0 +1,174 @@
+package FS::queue;
+
+use strict;
+use vars qw( @ISA );
+use FS::Record qw( qsearch qsearchs dbh );
+use FS::queue_arg;
+
+@ISA = qw(FS::Record);
+
+=head1 NAME
+
+FS::queue - Object methods for queue records
+
+=head1 SYNOPSIS
+
+  use FS::queue;
+
+  $record = new FS::queue \%hash;
+  $record = new FS::queue { 'column' => 'value' };
+
+  $error = $record->insert;
+
+  $error = $new_record->replace($old_record);
+
+  $error = $record->delete;
+
+  $error = $record->check;
+
+=head1 DESCRIPTION
+
+An FS::queue object represents an queued job.  FS::queue inherits from
+FS::Record.  The following fields are currently supported:
+
+=over 4
+
+=item jobnum - primary key
+
+=item job - fully-qualified subroutine name
+
+=item status - job status
+
+=back
+
+=head1 METHODS
+
+=over 4
+
+=item new HASHREF
+
+Creates a new job.  To add the example to the database, see L<"insert">.
+
+Note that this stores the hash reference, not a distinct copy of the hash it
+points to.  You can ask the object for a copy with the I<hash> method.
+
+=cut
+
+# the new method can be inherited from FS::Record, if a table method is defined
+
+sub table { 'queue'; }
+
+=item insert [ ARGUMENT, ARGUMENT... ]
+
+Adds this record to the database.  If there is an error, returns the error,
+otherwise returns false.
+
+If any arguments are supplied, a queue_arg record for each argument is also
+created (see L<FS::queue_arg>).
+
+=cut
+
+sub insert {
+  my $self = shift;
+
+  local $SIG{HUP} = 'IGNORE';
+  local $SIG{INT} = 'IGNORE';
+  local $SIG{QUIT} = 'IGNORE';
+  local $SIG{TERM} = 'IGNORE';
+  local $SIG{TSTP} = 'IGNORE';
+  local $SIG{PIPE} = 'IGNORE';
+
+  my $oldAutoCommit = $FS::UID::AutoCommit;
+  local $FS::UID::AutoCommit = 0;
+  my $dbh = dbh;
+
+  my $error = $self->SUPER::insert;
+  if ( $error ) {
+    $dbh->rollback if $oldAutoCommit;
+    return $error;
+  }
+
+  foreach my $arg ( @_ ) {
+    my $queue_arg = new FS::queue_arg ( {
+      'jobnum' => $self->jobnum,
+      'arg'    => $arg,
+    } );
+    $error = $queue_arg->insert;
+    if ( $error ) {
+      $dbh->rollback if $oldAutoCommit;
+      return $error;
+    }
+  }
+
+  $dbh->commit or die $dbh->errstr if $oldAutoCommit;
+
+  '';
+
+}
+
+=item delete
+
+Delete this record from the database.
+
+=cut
+
+# the delete method can be inherited from FS::Record
+
+=item replace OLD_RECORD
+
+Replaces the OLD_RECORD with this one in the database.  If there is an error,
+returns the error, otherwise returns false.
+
+=cut
+
+# the replace method can be inherited from FS::Record
+
+=item check
+
+Checks all fields to make sure this is a valid job.  If there is
+an error, returns the error, otherwise returns false.  Called by the insert
+and replace methods.
+
+=cut
+
+sub check {
+  my $self = shift;
+  my $error =
+    $self->ut_numbern('jobnum')
+    || $self->ut_anything('job')
+    || $self->ut_numbern('_date')
+    || $self->ut_enum('status',['', qw( new locked failed )])
+  ;
+  return $error if $error;
+
+  $self->status('new') unless $self->status;
+  $self->_date(time) unless $self->_date;
+
+  ''; #no error
+}
+
+=item args
+
+=cut
+
+sub args {
+  my $self = shift;
+  map $_->arg, qsearch( 'queue_arg', { 'jobnum' => $self->jobnum } );
+}
+
+=back
+
+=head1 VERSION
+
+$Id: queue.pm,v 1.1 2001-09-11 00:08:18 ivan Exp $
+
+=head1 BUGS
+
+=head1 SEE ALSO
+
+L<FS::Record>, schema.html from the base documentation.
+
+=cut
+
+1;
+
diff --git a/FS/FS/queue_arg.pm b/FS/FS/queue_arg.pm
new file mode 100644
index 0000000..08fe473
--- /dev/null
+++ b/FS/FS/queue_arg.pm
@@ -0,0 +1,121 @@
+package FS::queue_arg;
+
+use strict;
+use vars qw( @ISA );
+use FS::Record qw( qsearch qsearchs );
+
+@ISA = qw(FS::Record);
+
+=head1 NAME
+
+FS::queue_arg - Object methods for queue_arg records
+
+=head1 SYNOPSIS
+
+  use FS::queue_arg;
+
+  $record = new FS::queue_arg \%hash;
+  $record = new FS::queue_arg { 'column' => 'value' };
+
+  $error = $record->insert;
+
+  $error = $new_record->replace($old_record);
+
+  $error = $record->delete;
+
+  $error = $record->check;
+
+=head1 DESCRIPTION
+
+An FS::queue_arg object represents job argument.  FS::queue_arg inherits from
+FS::Record.  The following fields are currently supported:
+
+=over 4
+
+=item argnum - primary key
+
+=item jobnum - see L<FS::queue>
+
+=item arg - argument
+
+=back
+
+=head1 METHODS
+
+=over 4
+
+=item new HASHREF
+
+Creates a new argument.  To add the example to the database, see L<"insert">.
+
+Note that this stores the hash reference, not a distinct copy of the hash it
+points to.  You can ask the object for a copy with the I<hash> method.
+
+=cut
+
+# the new method can be inherited from FS::Record, if a table method is defined
+
+sub table { 'queue_arg'; }
+
+=item insert
+
+Adds this record to the database.  If there is an error, returns the error,
+otherwise returns false.
+
+=cut
+
+# the insert method can be inherited from FS::Record
+
+=item delete
+
+Delete this record from the database.
+
+=cut
+
+# the delete method can be inherited from FS::Record
+
+=item replace OLD_RECORD
+
+Replaces the OLD_RECORD with this one in the database.  If there is an error,
+returns the error, otherwise returns false.
+
+=cut
+
+# the replace method can be inherited from FS::Record
+
+=item check
+
+Checks all fields to make sure this is a valid argument.  If there is
+an error, returns the error, otherwise returns false.  Called by the insert
+and replace methods.
+
+=cut
+
+sub check {
+  my $self = shift;
+  my $error =
+    $self->ut_numbern('argnum')
+    || $self->ut_numbern('jobnum')
+    || $self->ut_anything('arg')
+  ;
+  return $error if $error;
+
+  ''; #no error
+}
+
+=back
+
+=head1 VERSION
+
+$Id: queue_arg.pm,v 1.1 2001-09-11 00:08:18 ivan Exp $
+
+=head1 BUGS
+
+=head1 SEE ALSO
+
+L<FS::queue>, L<FS::Record>, schema.html from the base documentation.
+
+=cut
+
+1;
+
diff --git a/FS/MANIFEST b/FS/MANIFEST
index ae3e0b0..69123db 100644
--- a/FS/MANIFEST
+++ b/FS/MANIFEST
@@ -6,6 +6,7 @@ README
 bin/freeside-bill
 bin/freeside-email
 bin/freeside-print-batch
+bin/freeside-queued
 FS.pm
 FS/CGI.pm
 FS/Conf.pm
@@ -51,6 +52,8 @@ FS/svc_www.pm
 FS/CGIwrapper.pm
 FS/svc_forward.pm
 FS/raddb.pm
+FS/queue.pm
+FS/queue_arg.pm
 t/agent.t
 t/agent_type.t
 t/CGI.t
@@ -89,5 +92,7 @@ t/svc_domain.t
 t/svc_forward.t
 t/svc_www.t
 t/type_pkgs.t
+t/queue.t
+t/queue_arg.t
 t/UID.t
 t/raddb.t
diff --git a/FS/bin/freeside-bill b/FS/bin/freeside-bill
index 1dce9a1..82b3321 100755
--- a/FS/bin/freeside-bill
+++ b/FS/bin/freeside-bill
@@ -1,3 +1,5 @@
+#!/usr/bin/perl -w
+# don't take any world-facing input
 #!/usr/bin/perl -Tw
 
 use strict;
@@ -121,7 +123,7 @@ customers.  Otherwise, bills all customers.
 
 =head1 VERSION
 
-$Id: freeside-bill,v 1.8 2001-09-03 22:07:39 ivan Exp $
+$Id: freeside-bill,v 1.9 2001-09-11 00:08:18 ivan Exp $
 
 =head1 BUGS
 
diff --git a/FS/bin/freeside-queued b/FS/bin/freeside-queued
new file mode 100644
index 0000000..5acffb5
--- /dev/null
+++ b/FS/bin/freeside-queued
@@ -0,0 +1,145 @@
+#!/usr/bin/perl -w
+
+use strict;
+use Fcntl qw(:flock);
+use POSIX qw(setsid);
+use FS::UID qw(adminsuidsetup);
+use FS::Record qw(qsearchs);
+use FS::queue;
+
+# no autoloading just yet
+use FS::cust_main;
+
+my $pid_file = '/var/run/freeside-queued.pid';
+
+$SIG{CHLD} = sub { wait }; #zombie prevention
+
+my $sigterm = 0;
+my $sigint = 0;
+$SIG{INT} = sub { warn "SIGINT received; shutting down\n"; $sigint++; };
+$SIG{TERM} = sub { warn "SIGTERM received; shutting down\n"; $sigterm++; };
+
+my $user = shift or die &usage;
+
+&daemonize;
+
+my $log_file = "/usr/local/etc/freeside/queuelog.";
+
+$> = $FS::UID::freeside_uid unless $>;
+adminsuidsetup $user;
+
+$log_file = "/usr/local/etc/freeside/queuelog.". $FS::UID::datasrc;
+
+$SIG{__DIE__} = \&_die;
+$SIG{__WARN__} = \&_logmsg;
+
+
+while (1) {
+
+  my $job = qsearchs(
+    'queue',
+    { 'status' => 'new' },
+    '',
+    'ORDER BY jobnum FOR UPDATE LIMIT 1'
+  ) or do {
+    sleep 5;
+    next;
+  };
+
+  my %hash = $job->hash;
+  $hash{'status'} = 'locked';
+  my $ljob = new FS::queue ( \%hash );
+  my $error = $ljob->replace($job);
+  die $error if $error;
+
+  my @args = $ljob->args;
+
+  #fork a child for each job (up to some maximum perhaps?)
+  #single-threaded for now.
+
+  my $eval = "&". $ljob->job. '(@args);';
+  warn "running $eval";
+  eval $eval;
+  if ( $@ ) {
+    warn "job $eval failed";
+    my $hash = $ljob->hash;
+    $hash{'status'} = 'failed';
+    my $fjob = new FS::queue( \%hash );
+    my $error = $fjob->replace($ljob);
+    die $error if $error;
+  } else {
+    $ljob->delete;
+  }
+
+} continue {
+  if ( $sigterm ) {
+    warn "received TERM signal; exiting\n";
+    exit;
+  }
+  if ( $sigint ) {
+    warn "received INT signal; exiting\n";
+    exit;
+  }
+}
+
+
+sub datestamp {
+  time2str("%m%d%Y", time);
+}
+
+sub _die {
+  my $msg = shift;
+  unlink $pid_file if -e $pid_file;
+  _logmsg($msg);
+}
+
+sub _logmsg {
+  chomp( my $msg = shift );
+  my $log = new IO::File ">>$log_file";
+  flock($log, LOCK_EX);
+  seek($log, 0, 2);
+  print $log "[". time2str("%a %b %e %T %Y",time). "] [$$] $msg\n";
+  flock($log, LOCK_UN);
+}
+
+sub daemonize {
+
+  chdir "/" or die "Can't chdir to /: $!";
+  open STDIN, '/dev/null'   or die "Can't read /dev/null: $!";
+  defined(my $pid = fork) or die "Can't fork: $!";
+  if ( $pid ) {
+    print "freeside-queued started with pid $pid\n"; #logging to $log_file\n";
+    exit unless $pid_file;
+    my $pidfh = new IO::File ">$pid_file" or exit;
+    print $pidfh "$pid\n";
+    exit;
+  }
+  open STDOUT, '>/dev/null'
+                            or die "Can't write to /dev/null: $!";
+  setsid                  or die "Can't start a new session: $!";
+  open STDERR, '>&STDOUT' or die "Can't dup stdout: $!";
+
+}
+
+=head1 NAME
+
+freeside-queued - Job queue daemon
+
+=head1 SYNOPSIS
+
+  freeside-queued user
+
+=head1 DESCRIPTION
+
+Job queue daemon.  Should be running at all times.
+
+user: from the mapsecrets file - see config.html from the base documentation
+
+=head1 VERSION
+
+=head1 BUGS
+
+=head1 SEE ALSO
+
+=cut
+
diff --git a/FS/t/queue.t b/FS/t/queue.t
new file mode 100644
index 0000000..43e3373
--- /dev/null
+++ b/FS/t/queue.t
@@ -0,0 +1,5 @@
+BEGIN { $| = 1; print "1..1\n" }
+END {print "not ok 1\n" unless $loaded;}
+use FS::queue;
+$loaded=1;
+print "ok 1\n";
diff --git a/FS/t/queue_arg.t b/FS/t/queue_arg.t
new file mode 100644
index 0000000..cf3f91d
--- /dev/null
+++ b/FS/t/queue_arg.t
@@ -0,0 +1,5 @@
+BEGIN { $| = 1; print "1..1\n" }
+END {print "not ok 1\n" unless $loaded;}
+use FS::queue_arg;
+$loaded=1;
+print "ok 1\n";
diff --git a/README.1.4.0pre2-3 b/README.1.4.0pre2-3
index 23b838e..1de2a54 100644
--- a/README.1.4.0pre2-3
+++ b/README.1.4.0pre2-3
@@ -10,7 +10,28 @@ CREATE TABLE part_svc_column (
 
 CREATE UNIQUE INDEX part_svc_column1 ON part_svc_column ( svcpart, columnname );
 
+CREATE TABLE queue (
+  jobnum int primary key,
+  job varchar not null,
+  _date int not null,
+  status varchar(80) not null
+);
+
+CREATE TABLE queue_arg (
+  argnum int primary key,
+  jobnum int not null,
+  arg varchar null
+);
+
+CREATE INDEX queue_arg1 ON queue_arg ( jobnum );
+
 Run bin/fs-migrate-part_svc
 
 Run bin/dbdef-create
 
+Create the `/usr/local/etc/freeside/cache.<i>datasrc</i>' directory
+  (ownded by the freeside user).
+
+freeside-queued was installed with the Perl modules.  Start it now and ensure
+that it is run upon system startup.
+
diff --git a/htetc/handler.pl b/htetc/handler.pl
index 145529a..b6f6c69 100644
--- a/htetc/handler.pl
+++ b/htetc/handler.pl
@@ -63,8 +63,8 @@ sub handler
       use FS::CGI qw(header menubar popurl table);
 
       $cgi = new CGI;
-      #&cgisuidsetup($cgi);
-      &cgisuidsetup($r);
+      &cgisuidsetup($cgi);
+      #&cgisuidsetup($r);
       $p = popurl(2);
     }
     $r->content_type('text/html');
diff --git a/httemplate/browse/part_svc.cgi b/httemplate/browse/part_svc.cgi
index cfb1ad8..8a564ba 100755
--- a/httemplate/browse/part_svc.cgi
+++ b/httemplate/browse/part_svc.cgi
@@ -1,4 +1,4 @@
-<!-- $Id: part_svc.cgi,v 1.4 2001-09-06 20:41:59 ivan Exp $ -->
+<!-- $Id: part_svc.cgi,v 1.5 2001-09-11 00:08:18 ivan Exp $ -->
 <%= header('Service Definition Listing', menubar( 'Main Menu' => $p) ) %>
 
     Services are items you offer to your customers.<BR><BR>
@@ -35,7 +35,7 @@
      foreach my $field ( @fields ) {
        my $flag = $part_svc->part_svc_column($field)->columnflag;
 %>
-     <%= $n1 %><TD><%= $row %></TD><TD>
+     <%= $n1 %><TD><%= $field %></TD><TD>
 
 <%     if ( $flag eq "D" ) { print "Default"; }
          elsif ( $flag eq "F" ) { print "Fixed"; }
diff --git a/httemplate/docs/install.html b/httemplate/docs/install.html
index 1529c64..4bf7b44 100644
--- a/httemplate/docs/install.html
+++ b/httemplate/docs/install.html
@@ -109,9 +109,11 @@ require "/usr/local/apache/conf/handler.pl";
 <ul>
 <li>Restrict access to this web interface.  (with <a href="http://httpd.apache.org/docs/misc/FAQ.html#user-authentication">Apache</a>)
 <li>Create the necessary <a href="config.html">configuration files</a>.
-<li>Create the `/usr/local/etc/freeside/counters.<i>datasrc</i>', and 
+<li>Create the `/usr/local/etc/freeside/counters.<i>datasrc</i>', 
+  `/usr/local/etc/freeside/cache.<i>datasrc</i>', and 
   `/usr/local/etc/freeside/export.<i>datasrc</i>' directories for each <i>datasrc</i> (owned by the freeside user).
   <li>As the freeside user, run bin/fs-setup to create the database tables.
+  <li>freeside-queued was installed with the Perl modules.  Start it now and ensure that is run upon system startup.
   <li>Now proceed to the initial <a href="admin.html">administration</a> of your installation.
 </ul>
 </body>
diff --git a/httemplate/docs/schema.html b/httemplate/docs/schema.html
index 32c365e..e3fe980 100644
--- a/httemplate/docs/schema.html
+++ b/httemplate/docs/schema.html
@@ -43,6 +43,14 @@
         <li>otaker - order taker
         <li>reason
       </ul>
+    <li><a name="cust_credit_bill" href="man/FS/cust_credit_bill.html">cust_credit_bill</a> - Credit invoice application.  Links a credit to an invoice.
+      <ul>
+        <li>creditbillnum - primary key
+        <li>crednum - <a href="#cust_credit">credit</a> being applied
+        <li>invnum - <a href="#cust_bill">invoice</a> to which credit is applied
+        <li>amount - amount applied
+        <li>_date
+      </ul>
     <li><a name="cust_main" href="man/FS/cust_main.html">cust_main</a> - Customers
       <ul>
         <li>custnum - primary key
@@ -81,6 +89,8 @@
         <li>payname - billing name (name on card)
         <li>tax - tax exempt, Y or null
         <li>otaker - order taker
+        <li>referral_custnum
+        <li>comments
       </ul>
       (columns in <i>italics</i> are optional)
     <li><a name="cust_main_invoice" href="man/FS/cust_main_invoice.html">cust_main_invoice</a> - Invoice destinations for email invoices.  Note that a customer can have many email destinations for their invoice (either literal or via svcnum), but only one postal destination.
@@ -302,5 +312,18 @@
         <li>typenum - <a href="#agent_type">agent type</a>
         <li>pkgpart - <a href="#part_pkg">Package definition</a>
       </ul>
+    <li><a name="queue" href="man/FS/queue.html">queue</a> - job queue
+      <ul>
+        <li>jobnum - primary key
+        <li>job
+        <li>_date
+        <li>status
+      </ul>
+    <li><a name="queue_arg" href="man/FS/queue_arg.html">queue_arg</a> - job arguments
+      <ul>
+        <li>argnum - primary key
+        <li>jobnum - <a href="#queue">job</a>
+        <li>arg - argument
+      </ul>
   </ul>
 </body>
diff --git a/httemplate/docs/upgrade8.html b/httemplate/docs/upgrade8.html
index 67b0e5f..d86adc0 100644
--- a/httemplate/docs/upgrade8.html
+++ b/httemplate/docs/upgrade8.html
@@ -103,6 +103,19 @@ CREATE TABLE part_svc_column (
   columnflag char(1) null
 );
 
+CREATE TABLE queue (
+  jobnum int primary key,
+  job varchar not null,
+  _date int not null,
+  status varchar(80) not null
+);
+
+CREATE TABLE queue_arg (
+  argnum int primary key,
+  jobnum int not null,
+  arg varchar null
+);
+CREATE INDEX queue_arg1 ON queue_arg ( jobnum );
 
 ALTER TABLE svc_acct ADD domsvc integer NOT NULL;
 ALTER TABLE svc_domain ADD catchall integer NULL;
@@ -217,5 +230,9 @@ ALTER TABLE cust_refund DROP COLUMN crednum;
 </pre></font>
 </td></tr></table>
   <li><b>IMPORTANT: After applying the second set of database changes</b>, run bin/dbdef-create again.
-  <li>create the conf.dbsrc/user_policy as appropriate for your site
+  <li>create the <a href="config.html#username_policy">user_policy configuration file</a> as appropriate for your site.
+  <li>Create the `/usr/local/etc/freeside/cache.<i>datasrc</i>' directory
+      (ownded by the freeside user).
+  <li>freeside-queued was installed with the Perl modules.  Start it now and ensure that is run upon system startup.
+</ul>
 </body>
diff --git a/httemplate/edit/part_svc.cgi b/httemplate/edit/part_svc.cgi
index c0d632f..c0a7896 100755
--- a/httemplate/edit/part_svc.cgi
+++ b/httemplate/edit/part_svc.cgi
@@ -1,4 +1,4 @@
-<!-- $Id: part_svc.cgi,v 1.7 2001-09-06 20:42:00 ivan Exp $ -->
+<!-- $Id: part_svc.cgi,v 1.8 2001-09-11 00:08:18 ivan Exp $ -->
 <% 
    my $part_svc;
    if ( $cgi->param('error') ) { #error
@@ -147,7 +147,7 @@ foreach my $svcdb ( qw(
 
   my(@fields) = $svcdb eq 'konq_kludge'
                   ? ()
-                  : grep { $_ ne 'svcnum' } fields($svcdb) );
+                  : grep { $_ ne 'svcnum' } fields($svcdb);
   #my($rowspan)=scalar(@rows);
 
   #my($ptmp)="<TD ROWSPAN=$rowspan>$svcdb</TD>";
@@ -180,7 +180,7 @@ function fixup(what) {
     my $part_svc_column = $part_svc->part_svc_column($field);
     my $value = $cgi->param('error')
                   ? $cgi->param("${svcdb}__${field}")
-                  : $$part_svc_column->columnvalue;
+                  : $part_svc_column->columnvalue;
     my $flag = $cgi->param('error')
                  ? $cgi->param("${svcdb}__${field}_flag")
                  : $part_svc_column->columnflag;
diff --git a/httemplate/edit/process/part_svc.cgi b/httemplate/edit/process/part_svc.cgi
index 5652c58..937f7fb 100755
--- a/httemplate/edit/process/part_svc.cgi
+++ b/httemplate/edit/process/part_svc.cgi
@@ -17,7 +17,7 @@ my $new = new FS::part_svc ( {
 
 my $error;
 if ( $svcpart ) {
-  $error = $new->replace($old);
+  $error = $new->replace($old, '1.3-COMPAT');
 } else {
   $error = $new->insert;
   $svcpart=$new->getfield('svcpart');
diff --git a/httemplate/index.html b/httemplate/index.html
index f24d354..2fbc062 100644
--- a/httemplate/index.html
+++ b/httemplate/index.html
@@ -60,6 +60,7 @@
               <LI>unlinked domains (<A HREF="search/svc_domain.cgi?UN_svcnum">by service number</A>) (<A HREF="search/svc_domain.cgi?UN_domain">by domain</A>)
             </UL>
           <LI><A HREF="browse/nas.cgi">NAS ports</A>
+          <LI><A HREF="browse/queue.cgi">Joe queue</A>
         </ul>
       <li><A NAME="admin">Administration</a>
         <ul>
diff --git a/httemplate/search/cust_main.cgi b/httemplate/search/cust_main.cgi
index 5defaf4..9dcada7 100755
--- a/httemplate/search/cust_main.cgi
+++ b/httemplate/search/cust_main.cgi
@@ -1,5 +1,5 @@
 <%
-#<!-- $Id: cust_main.cgi,v 1.4 2001-08-28 16:58:08 ivan Exp $ -->
+#<!-- $Id: cust_main.cgi,v 1.5 2001-09-11 00:08:18 ivan Exp $ -->
 
 use strict;
 #use vars qw( $conf %ncancelled_pkgs %all_pkgs $cgi @cust_main $sortby );
@@ -253,13 +253,12 @@ sub lastsearch {
 
   } else {
 
-    my(%last);
+    &FS::cust_main::check_and_rebuild_fuzzyfiles;
+    my $all_last = &FS::cust_main::all_last;
 
-    my(@all_last)=map $_->getfield('last'), qsearch('cust_main',{});
-    push @all_last, grep $_, map $_->getfield('ship_last'), qsearch('cust_main',{})
-      if defined dbdef->table('cust_main')->column('ship_last');
+    my %last;
     if ($last_type{'Fuzzy'}) { 
-      foreach ( amatch($last, [ qw(i) ], @all_last) ) {
+      foreach ( amatch($last, [ qw(i) ], @$all_last) ) {
         $last{$_}++; 
       }
     }
@@ -300,13 +299,12 @@ sub companysearch {
 
   } else {
 
-    my(%company);
-    my(@all_company)=map $_->company, qsearch('cust_main',{});
-    push @all_company, grep $_, map $_->getfield('ship_company'), qsearch('cust_main',{})
-      if defined dbdef->table('cust_main')->column('ship_last');
+    &FS::cust_main::check_and_rebuild_fuzzyfiles;
+    my $all_company = &FS::cust_main::all_company;
 
+    my %company;
     if ($company_type{'Fuzzy'}) { 
-      foreach ( amatch($company, [ qw(i) ], @all_company ) ) {
+      foreach ( amatch($company, [ qw(i) ], @$all_company ) ) {
         $company{$_}++;
       }
     }