X-Git-Url: http://git.freeside.biz/gitweb/?p=freeside.git;a=blobdiff_plain;f=FS%2Fbin%2Ffreeside-upgrade;h=77087c3737ef925988fcce95ac46f40a90d4c054;hp=285ff7a0e16b3efc864db6958d7930210b7fd511;hb=5e9be05e2abc350720b3c2031a8b7a0d39a226b8;hpb=673b9a458d9138523026963df6fa3b4683e09bae

diff --git a/FS/bin/freeside-upgrade b/FS/bin/freeside-upgrade
index 285ff7a0e..77087c373 100755
--- a/FS/bin/freeside-upgrade
+++ b/FS/bin/freeside-upgrade
@@ -1,85 +1,327 @@
 #!/usr/bin/perl -w
 
 use strict;
-use vars qw($DEBUG $DRY_RUN);
-use Term::ReadKey;
-use DBIx::DBSchema 0.27;
-use FS::UID qw(adminsuidsetup checkeuid datasrc );  #getsecrets);
+use vars qw( $opt_d $opt_s $opt_q $opt_v $opt_r $opt_c $opt_j $opt_a );
+use vars qw( $DEBUG $DRY_RUN );
+use Getopt::Std;
+use DBD::Pg qw(:async); #for -a
+use DBIx::DBSchema 0.46;
+use FS::UID qw(adminsuidsetup checkeuid datasrc driver_name);
+use FS::CurrentUser;
 use FS::Schema qw( dbdef dbdef_dist reload_dbdef );
+use FS::Misc::prune qw(prune_applications);
+use FS::Conf;
+use FS::Record qw(qsearch);
+use FS::Upgrade qw(upgrade_schema upgrade_config upgrade upgrade_sqlradius);
 
+my $start = time;
 
-$DEBUG = 1;
-$DRY_RUN = 0;
+die "Not running uid freeside!" unless checkeuid();
 
+getopts("dqrcsja");
 
-die "Not running uid freeside!" unless checkeuid();
+$DEBUG = !$opt_q;
+#$DEBUG = $opt_v;
+
+$DRY_RUN = $opt_d;
 
 my $user = shift or die &usage;
+$FS::CurrentUser::upgrade_hack = 1;
+$FS::UID::callback_hack = 1;
 my $dbh = adminsuidsetup($user);
+$FS::UID::callback_hack = 0;
+
+# pass command line opts through to upgrade* routines
+my %upgrade_opts = (
+  quiet   => $opt_q,
+  verbose => $opt_v,
+  queue   => $opt_j,
+  # others?
+);
+
+if ( driver_name =~ /^mysql/i ) { #until 0.39 is required above
+  eval "use DBIx::DBSchema 0.39;";
+  die $@ if $@;
+}
 
 #needs to match FS::Schema...
-my $dbdef_file = "/usr/local/etc/freeside/dbdef.". datasrc;
+my $dbdef_file = "%%%FREESIDE_CONF%%%/dbdef.". datasrc;
 
 dbdef_create($dbh, $dbdef_file);
+
+delete $FS::Schema::dbdef_cache{$dbdef_file}; #force an actual reload
 reload_dbdef($dbdef_file);
 
+warn "Upgrade startup completed in ". (time-$start). " seconds\n"; # if $DEBUG;
+$start = time;
 
-foreach my $table ( dbdef_dist->tables ) {
+#$DBIx::DBSchema::DEBUG = $DEBUG;
+#$DBIx::DBSchema::Table::DEBUG = $DEBUG;
+#$DBIx::DBSchema::Index::DEBUG = $DEBUG;
 
-  if ( dbdef->table($table) ) {
+my @bugfix = ();
 
-    warn "$table exists\n" if $DEBUG > 1;
+if ( $DRY_RUN ) {
+  print join(";\n", @bugfix ). ";\n";
+} else {
+  foreach my $statement ( @bugfix ) {
+    warn "$statement\n";
+    $dbh->do( $statement )
+      or die "Error: ". $dbh->errstr. "\n executing: $statement";
+  }
+}
 
-    foreach my $column ( dbdef_dist->table($table)->columns ) {
-      if ( dbdef->table($table)->column($column) )  {
-        warn "  $table.$column exists\n" if $DEBUG > 2;
-      } else {
+###
+# Fixes before schema upgrade
+###
+# this isn't actually the main schema upgrade, this calls _upgrade_schema
+# in any class that has it
+if ( $DRY_RUN ) {
+  #XXX no dry run for upgrade_schema stuff yet.
+  # looking at the code some are a mix of SQL statements and our methods, icky.
+  # its not like dry run is 100% anyway, all sort of other later upgrade tasks
+  # aren't printed either
+} else {
+  upgrade_schema(%upgrade_opts);
+
+  dbdef_create($dbh, $dbdef_file);
+  delete $FS::Schema::dbdef_cache{$dbdef_file}; #force an actual reload
+  reload_dbdef($dbdef_file);
+}
 
-        if ( $DEBUG ) {
-          print STDERR "column $table.$column does not exist.  create?";
-          next unless yesno();
-        }
+###
+# Now here is the main/automatic schema upgrade via DBIx::DBSchema
+###
 
-        foreach my $statement (
-          dbdef_dist->table($table)->column($column)->sql_add_column( $dbh )
-        ) {
-          warn "$statement\n" if $DEBUG || $DRY_RUN;
-          unless ( $DRY_RUN ) {
-            $dbh->do( $statement)
-              or die "CREATE error: ". $dbh->errstr. "\nexecuting: $statement";
-          }
-        }
+my $conf = new FS::Conf;
 
-      }
+my $dbdef_dist = dbdef_dist(
+  datasrc,
+  { 'queue-no_history' => $conf->exists('queue-no_history') },
+);
 
-    }
+my @statements = dbdef->sql_update_schema( $dbdef_dist,
+                                           $dbh,
+                                           { 'nullify_default' => 1, },
+                                         );
+
+###
+# New custom fields
+###
+# 1. prevent new custom field columns from being dropped by upgrade
+# 2. migrate old virtual fields to real fields (new custom fields)
+
+my $cfsth = $dbh->prepare("SELECT * FROM part_virtual_field") 
+                                                         or die $dbh->errstr;
+$cfsth->execute or die $cfsth->errstr;
+my $cf; 
+while ( $cf = $cfsth->fetchrow_hashref ) {
+    my $tbl = $cf->{'dbtable'};
+    my $name = $cf->{'name'};
+    $name = lc($name) unless driver_name =~ /^mysql/i;
+
+    @statements = grep { $_ !~ /^\s*ALTER\s+TABLE\s+(h_|)$tbl\s+DROP\s+COLUMN\s+cf_$name\s*$/i }
+                                                                    @statements;
+    push @statements, 
+        "ALTER TABLE $tbl ADD COLUMN cf_$name varchar(".$cf->{'length'}.")"
+     unless (dbdef->table($tbl) && dbdef->table($tbl)->column("cf_$name"));
+    push @statements, 
+        "ALTER TABLE h_$tbl ADD COLUMN cf_$name varchar(".$cf->{'length'}.")"
+     unless (dbdef->table("h_$tbl") && dbdef->table("h_$tbl")->column("cf_$name"));
+}
+warn "Custom fields schema upgrade completed";
+
+###
+# Other stuff
+###
+
+@statements = 
+  grep { $_ !~ /^CREATE +INDEX +h_queue/i } #useless, holds up queue insertion
+       @statements;
+
+unless ( driver_name =~ /^mysql/i ) {
+  #not necessary under non-mysql, takes forever on big db
+  @statements =
+    grep { $_ !~ /^ *ALTER +TABLE +h_queue +ALTER +COLUMN +job +TYPE +varchar\(512\) *$/i }
+         @statements;
+}
 
-    #should eventually check & create missing indices
+if ( $opt_c ) {
 
-    #should eventually drop columns not in dbdef_dist...
+  #can always add it back for 4.x->4.x if we need it
+  die "FATAL: -c removed: cdr / h_cdr upgrade is required for 4.x\n";
 
-  } else {
+  @statements =
+    grep { $_ !~ /^ *ALTER +TABLE +(h_)?cdr /i }
+         @statements;
 
-    if ( $DEBUG ) {
-      print STDERR "table $table does not exist.  create?";
-      next unless yesno();
+  @statements =
+    grep { $_ !~ /^ *CREATE +INDEX +(h_)?cdr\d+ /i }
+         @statements;
+
+}
+
+
+###
+# Now run the @statements
+###
+
+if ( $DRY_RUN ) {
+  print
+    join(";\n", @statements ). ";\n";
+  exit;
+} elsif ( $opt_a ) {
+
+  ###
+  # -a: Run schema changes in parallel (Pg only).
+  ###
+
+  my $MAX_HANDLES; # undef for now, set it if you want a limit
+
+  my @phases = map { [] } 0..4;
+  my $fsupgrade_idx = 1;
+  my %idx_map;
+  foreach (@statements) {
+    if ( /^ *(CREATE|ALTER) +TABLE/ ) {
+      # phase 0: CREATE TABLE, ALTER TABLE
+      push @{ $phases[0] }, $_;
+    } elsif ( /^ *ALTER +INDEX.* RENAME TO dbs_temp(\d+)/ ) {
+      # phase 1: rename index to dbs_temp%d
+      # (see DBIx::DBSchema::Table)
+      # but in this case, uniqueify all the dbs_temps.  This method only works
+      # because they are in the right order to begin with...
+      my $dbstemp_idx = $1;
+      s/dbs_temp$dbstemp_idx/fsupgrade_temp$fsupgrade_idx/;
+      $idx_map{ $dbstemp_idx } = $fsupgrade_idx;
+      push @{ $phases[1] }, $_;
+      $fsupgrade_idx++;
+    } elsif ( /^ *(CREATE|DROP)( +UNIQUE)? +INDEX/ ) {
+      # phase 2: create/drop indices
+      push @{ $phases[2] }, $_;
+    } elsif ( /^ *ALTER +INDEX +dbs_temp(\d+) +RENAME/ ) {
+      # phase 3: rename temp indices back to real ones
+      my $dbstemp_idx = $1;
+      my $mapped_idx = $idx_map{ $dbstemp_idx }
+        or die "unable to remap dbs_temp$1 RENAME statement";
+      s/dbs_temp$dbstemp_idx/fsupgrade_temp$mapped_idx/;
+      push @{ $phases[3] }, $_;
+    } else {
+      # phase 4: everything else (CREATE SEQUENCE, SELECT SETVAL, etc.)
+      push @{ $phases[4] }, $_;
     }
+  }
+  my $i = 0;
+  my @busy = ();
+  my @free = ();
+  foreach my $phase (@phases) {
+    warn "Starting schema changes, phase $i...\n";
+    while (@$phase or @busy) {
+      # check status of all running tasks
+      my @newbusy;
+      my $failed_clone;
+      for my $clone (@busy) {
+        if ( $clone->pg_ready ) {
+          # then clean it up
+          my $rv = $clone->pg_result && $clone->commit;
+          $failed_clone = $clone if !$rv;
+          push @free, $clone;
+        } else {
+          push @newbusy, $clone;
+        }
+      }
+      if ( $failed_clone ) {
+        my $errstr = $failed_clone->errstr;
+        foreach my $clone (@newbusy, $failed_clone) {
+          $clone->pg_cancel if $clone->{pg_async_status} == 1;
+          $clone->disconnect;
+        }
+        die "$errstr\n";
+      }
+      @busy = @newbusy;
+      if (my $statement = $phase->[0]) {
+        my $clone;
+        if ( @free ) {
+          $clone = shift(@free);
+        } elsif ( !$MAX_HANDLES or 
+                  scalar(@free) + scalar(@busy) < $MAX_HANDLES ) {
+          $clone = $dbh->clone; # this will fail if over the server limit
+        }
 
-    foreach my $statement (
-      dbdef_dist->table($table)->sql_create_table( $dbh )
-    ) {
-      warn "$statement\n" if $DEBUG || $DRY_RUN;
-      unless ( $DRY_RUN ) {
-        $dbh->do( $statement)
-          or die "CREATE error: ". $dbh->errstr. "\nexecuting: $statement";
+        if ( $clone ) {
+          my $rv = $clone->do($statement, {pg_async => PG_ASYNC});
+          if ( $rv ) {
+            warn "$statement\n";
+            shift @{ $phase }; # and actually take the statement off the queue
+            push @busy, $clone;
+          } # else I don't know, wait and retry
+        } # else too many handles, wait and retry
+      } elsif (@busy) {
+        # all statements are dispatched
+        warn "Waiting for phase $i to complete\n";
+        sleep 30;
       }
+    } # while @$phase or @busy
+    $i++;
+  } # foreach $phase
+  warn "Schema changes complete.\n";
+
+#  warn "Pre-schema change upgrades completed in ". (time-$start). " seconds\n"; # if $DEBUG;
+#  $start = time;
+
+#  dbdef->update_schema( dbdef_dist(datasrc), $dbh );
+
+} else {
+
+  ###
+  # normal case, run statements sequentially
+  ###
+
+  foreach my $statement ( @statements ) {
+    warn "$statement\n";
+    $dbh->do( $statement )
+      or die "Error: ". $dbh->errstr. "\n executing: $statement";
+  }
+}
+
+warn "Schema upgrade completed in ". (time-$start). " seconds\n"; # if $DEBUG;
+$start = time;
+
+my $hashref = {};
+$hashref->{dry_run} = 1 if $DRY_RUN;
+$hashref->{debug} = 1 if $DEBUG && $DRY_RUN;
+prune_applications($hashref) unless $opt_s;
+
+warn "Application pruning completed in ". (time-$start). " seconds\n"; # if $DEBUG;
+$start = time;
+
+print "\n" if $DRY_RUN;
+
+if ( $dbh->{Driver}->{Name} =~ /^mysql/i && ! $opt_s ) {
+
+  foreach my $table (qw( svc_acct svc_phone )) {
+
+    my $sth = $dbh->prepare(
+      "SELECT COUNT(*) FROM duplicate_lock WHERE lockname = '$table'"
+    ) or die $dbh->errstr;
+
+    $sth->execute or die $sth->errstr;
+
+    unless ( $sth->fetchrow_arrayref->[0] ) {
+
+      $sth = $dbh->prepare(
+        "INSERT INTO duplicate_lock ( lockname ) VALUES ( '$table' )"
+      ) or die $dbh->errstr;
+
+      $sth->execute or die $sth->errstr;
+
     }
 
   }
 
-}
+  warn "Duplication lock creation completed in ". (time-$start). " seconds\n"; # if $DEBUG;
+  $start = time;
 
-# should eventually drop tables not in dbdef_dist too i guess...
+}
 
 $dbh->commit or die $dbh->errstr;
 
@@ -87,33 +329,84 @@ dbdef_create($dbh, $dbdef_file);
 
 $dbh->disconnect or die $dbh->errstr;
 
-###
-
-my $all = 0;
-sub yesno {
-  print STDERR ' [yes/no/all] ';
-  if ( $all ) {
-    warn "yes\n";
-    return 1;
-  } else {
-    while ( 1 ) {
-      ReadMode 4;
-      my $x = lc(ReadKey);
-      ReadMode 0;
-      if ( $x eq 'n' ) {
-        warn "no\n"; 
-        return 0;
-      } elsif ( $x eq 'y' ) {
-        warn "yes\n"; 
-        return 1;
-      } elsif ( $x eq 'a' ) {
-        warn "yes\n"; 
-        $all = 1;
-        return 1;
-      }
+delete $FS::Schema::dbdef_cache{$dbdef_file}; #force an actual reload
+$FS::UID::AutoCommit = 0;
+$FS::UID::callback_hack = 1;
+$dbh = adminsuidsetup($user);
+$FS::UID::callback_hack = 0;
+unless ( $DRY_RUN || $opt_s ) {
+  my $dir = "%%%FREESIDE_CONF%%%/conf.". datasrc;
+  if (!scalar(qsearch('conf', {}))) {
+    my $error = FS::Conf::init_config($dir);
+    if ($error) {
+      warn "CONFIGURATION UPGRADE FAILED\n";
+      $dbh->rollback or die $dbh->errstr;
+      die $error;
     }
   }
 }
+$dbh->commit or die $dbh->errstr;
+$dbh->disconnect or die $dbh->errstr;
+
+$FS::UID::AutoCommit = 1;
+
+$dbh = adminsuidsetup($user);
+
+warn "Re-initialization with updated schema completed in ". (time-$start). " seconds\n"; # if $DEBUG;
+$start = time;
+
+#### NEW CUSTOM FIELDS:
+# 3. migrate old virtual field data to the new custom fields
+####
+$cfsth = $dbh->prepare("SELECT * FROM virtual_field left join part_virtual_field using (vfieldpart)")
+                                                         or die $dbh->errstr;
+$cfsth->execute or die $cfsth->errstr;
+my @cfst;
+while ( $cf = $cfsth->fetchrow_hashref ) {
+    my $tbl = $cf->{'dbtable'};
+    my $name = $cf->{'name'};
+    my $dtable = dbdef->table($tbl);
+    next unless $dtable && $dtable->primary_key; # XXX: warn first?
+    my $pkey = $dtable->primary_key;
+    next unless $dtable->column($pkey)->type =~ /int/i; # XXX: warn first?
+    push @cfst, "UPDATE $tbl set cf_$name = '".$cf->{'value'}."' WHERE $pkey = ".$cf->{'recnum'};
+    push @cfst, "DELETE FROM virtual_field WHERE vfieldnum = ".$cf->{'vfieldnum'};
+}
+foreach my $cfst ( @cfst ) {
+    warn "$cfst\n";
+    $dbh->do( $cfst )
+      or die "Error: ". $dbh->errstr. "\n executing: $cfst";
+}
+warn "Custom fields data upgrade completed";
+
+upgrade_config(%upgrade_opts)
+  unless $DRY_RUN || $opt_s;
+
+$dbh->commit or die $dbh->errstr;
+
+warn "Config updates completed in ". (time-$start). " seconds\n"; # if $DEBUG;
+$start = time;
+
+upgrade(%upgrade_opts)
+  unless $DRY_RUN || $opt_s;
+
+$dbh->commit or die $dbh->errstr;
+
+warn "Table updates completed in ". (time-$start). " seconds\n"; # if $DEBUG;
+$start = time;
+
+upgrade_sqlradius(%upgrade_opts)
+  unless $DRY_RUN || $opt_s || $opt_r;
+
+warn "SQL RADIUS updates completed in ". (time-$start). " seconds\n"; # if $DEBUG;
+$start = time;
+
+$dbh->commit or die $dbh->errstr;
+$dbh->disconnect or die $dbh->errstr;
+
+warn "Final commit and disconnection completed in ". (time-$start). " seconds; upgrade done!\n"; # if $DEBUG;
+
+###
 
 sub dbdef_create { # reverse engineer the schema from the DB and save to file
   my( $dbh, $file ) = @_;
@@ -122,8 +415,57 @@ sub dbdef_create { # reverse engineer the schema from the DB and save to file
 }
 
 sub usage {
-  die "Usage:\n  freeside-upgrade user\n"; 
+  die "Usage:\n  freeside-upgrade [ -d ] [ -q | -v ] [ -r ] [ -s ] [ -j ] [ -a ] user\n"; 
 }
 
-1;
+=head1 NAME
+
+freeside-upgrade - Upgrades database schema for new freeside verisons.
+
+=head1 SYNOPSIS
+
+  freeside-upgrade [ -d ] [ -q | -v ] [ -r ] [ -s ] [ -j ] [ -a ]
+
+=head1 DESCRIPTION
+
+Reads your existing database schema and updates it to match the current schema,
+adding any columns or tables necessary.
+
+Also performs other upgrade functions:
+
+=over 4
+
+=item Calls FS:: Misc::prune::prune_applications (probably unnecessary every upgrade, but simply won't find any records to change)
+
+=item If necessary, moves your configuration information from the filesystem in /usr/local/etc/freeside/conf.<datasrc> to the database.
+
+=back
+
+  [ -d ]: Dry run; output SQL statements (to STDOUT) only, but do not execute
+          them.
+
+  [ -q ]: Run quietly.  This may become the default at some point.
+
+  [ -v ]: Run verbosely, sending debugging information to STDERR.  This is the
+          current default.
+
+  [ -s ]: Schema changes only.  Useful for Pg/slony slaves where the data
+          changes will be replicated from the Pg/slony master.
+
+  [ -r ]: Skip sqlradius updates.  Useful for occassions where the sqlradius
+          databases may be inaccessible.
+
+  [ -j ]: Run certain upgrades asychronously from the job queue.  Currently 
+          used only for the 2.x -> 3.x cust_location, cust_pay and part_pkg
+          upgrades.  This may cause odd behavior before the upgrade is
+          complete, so it's recommended only for very large cust_main, cust_pay
+          and/or part_pkg tables that take too long to upgrade.
+
+  [ -a ]: Run schema changes in parallel (Pg only).  DBIx::DBSchema minimum 
+          version 0.41 recommended.  Recommended only for large databases and
+          powerful database servers, to reduce upgrade time.
+
+=head1 SEE ALSO
+
+=cut