4 use vars qw( $opt_d $opt_s $opt_q $opt_v $opt_r $opt_c $opt_j $opt_a );
5 use vars qw( $DEBUG $DRY_RUN );
7 use DBD::Pg qw(:async); #for -a
8 use DBIx::DBSchema 0.46;
9 use FS::UID qw(adminsuidsetup checkeuid datasrc driver_name);
11 use FS::Schema qw( dbdef dbdef_dist reload_dbdef );
12 use FS::Misc::prune qw(prune_applications);
14 use FS::Record qw(qsearch);
15 use FS::Upgrade qw(upgrade_schema upgrade_config upgrade upgrade_sqlradius);
19 die "Not running uid freeside!" unless checkeuid();
28 my $user = shift or die &usage;
29 $FS::CurrentUser::upgrade_hack = 1;
30 $FS::UID::callback_hack = 1;
31 my $dbh = adminsuidsetup($user);
32 $FS::UID::callback_hack = 0;
34 # pass command line opts through to upgrade* routines
42 if ( driver_name =~ /^mysql/i ) { #until 0.39 is required above
43 eval "use DBIx::DBSchema 0.39;";
47 #needs to match FS::Schema...
48 my $dbdef_file = "%%%FREESIDE_CONF%%%/dbdef.". datasrc;
50 dbdef_create($dbh, $dbdef_file);
52 delete $FS::Schema::dbdef_cache{$dbdef_file}; #force an actual reload
53 reload_dbdef($dbdef_file);
55 warn "Upgrade startup completed in ". (time-$start). " seconds\n"; # if $DEBUG;
58 #$DBIx::DBSchema::DEBUG = $DEBUG;
59 #$DBIx::DBSchema::Table::DEBUG = $DEBUG;
60 #$DBIx::DBSchema::Index::DEBUG = $DEBUG;
65 print join(";\n", @bugfix ). ";\n";
67 foreach my $statement ( @bugfix ) {
69 $dbh->do( $statement )
70 or die "Error: ". $dbh->errstr. "\n executing: $statement";
75 # Fixes before schema upgrade
77 # this isn't actually the main schema upgrade, this calls _upgrade_schema
78 # in any class that has it
80 #XXX no dry run for upgrade_schema stuff yet.
81 # looking at the code some are a mix of SQL statements and our methods, icky.
82 # its not like dry run is 100% anyway, all sort of other later upgrade tasks
83 # aren't printed either
85 upgrade_schema(%upgrade_opts);
87 dbdef_create($dbh, $dbdef_file);
88 delete $FS::Schema::dbdef_cache{$dbdef_file}; #force an actual reload
89 reload_dbdef($dbdef_file);
93 # Now here is the main/automatic schema upgrade via DBIx::DBSchema
96 my $conf = new FS::Conf;
98 my $dbdef_dist = dbdef_dist(
100 { 'queue-no_history' => $conf->exists('queue-no_history') },
103 my @statements = dbdef->sql_update_schema( $dbdef_dist,
105 { 'nullify_default' => 1, },
111 # 1. prevent new custom field columns from being dropped by upgrade
112 # 2. migrate old virtual fields to real fields (new custom fields)
114 my $cfsth = $dbh->prepare("SELECT * FROM part_virtual_field")
116 $cfsth->execute or die $cfsth->errstr;
118 while ( $cf = $cfsth->fetchrow_hashref ) {
119 my $tbl = $cf->{'dbtable'};
120 my $name = $cf->{'name'};
121 $name = lc($name) unless driver_name =~ /^mysql/i;
123 @statements = grep { $_ !~ /^\s*ALTER\s+TABLE\s+(h_|)$tbl\s+DROP\s+COLUMN\s+cf_$name\s*$/i }
126 "ALTER TABLE $tbl ADD COLUMN cf_$name varchar(".$cf->{'length'}.")"
127 unless (dbdef->table($tbl) && dbdef->table($tbl)->column("cf_$name"));
129 "ALTER TABLE h_$tbl ADD COLUMN cf_$name varchar(".$cf->{'length'}.")"
130 unless (dbdef->table("h_$tbl") && dbdef->table("h_$tbl")->column("cf_$name"));
132 warn "Custom fields schema upgrade completed";
139 grep { $_ !~ /^CREATE +INDEX +h_queue/i } #useless, holds up queue insertion
142 unless ( driver_name =~ /^mysql/i ) {
143 #not necessary under non-mysql, takes forever on big db
145 grep { $_ !~ /^ *ALTER +TABLE +h_queue +ALTER +COLUMN +job +TYPE +varchar\(512\) *$/i }
151 #can always add it back for 4.x->4.x if we need it
152 die "FATAL: -c removed: cdr / h_cdr upgrade is required for 4.x\n";
155 grep { $_ !~ /^ *ALTER +TABLE +(h_)?cdr /i }
159 grep { $_ !~ /^ *CREATE +INDEX +(h_)?cdr\d+ /i }
166 # Now run the @statements
171 join(";\n", @statements ). ";\n";
176 # -a: Run schema changes in parallel (Pg only).
179 my $MAX_HANDLES; # undef for now, set it if you want a limit
181 my @phases = map { [] } 0..4;
182 my $fsupgrade_idx = 1;
184 foreach (@statements) {
185 if ( /^ *(CREATE|ALTER) +TABLE/ ) {
186 # phase 0: CREATE TABLE, ALTER TABLE
187 push @{ $phases[0] }, $_;
188 } elsif ( /^ *ALTER +INDEX.* RENAME TO dbs_temp(\d+)/ ) {
189 # phase 1: rename index to dbs_temp%d
190 # (see DBIx::DBSchema::Table)
191 # but in this case, uniqueify all the dbs_temps. This method only works
192 # because they are in the right order to begin with...
193 my $dbstemp_idx = $1;
194 s/dbs_temp$dbstemp_idx/fsupgrade_temp$fsupgrade_idx/;
195 $idx_map{ $dbstemp_idx } = $fsupgrade_idx;
196 push @{ $phases[1] }, $_;
198 } elsif ( /^ *(CREATE|DROP)( +UNIQUE)? +INDEX/ ) {
199 # phase 2: create/drop indices
200 push @{ $phases[2] }, $_;
201 } elsif ( /^ *ALTER +INDEX +dbs_temp(\d+) +RENAME/ ) {
202 # phase 3: rename temp indices back to real ones
203 my $dbstemp_idx = $1;
204 my $mapped_idx = $idx_map{ $dbstemp_idx }
205 or die "unable to remap dbs_temp$1 RENAME statement";
206 s/dbs_temp$dbstemp_idx/fsupgrade_temp$mapped_idx/;
207 push @{ $phases[3] }, $_;
209 # phase 4: everything else (CREATE SEQUENCE, SELECT SETVAL, etc.)
210 push @{ $phases[4] }, $_;
216 foreach my $phase (@phases) {
217 warn "Starting schema changes, phase $i...\n";
218 while (@$phase or @busy) {
219 # check status of all running tasks
222 for my $clone (@busy) {
223 if ( $clone->pg_ready ) {
225 my $rv = $clone->pg_result && $clone->commit;
226 $failed_clone = $clone if !$rv;
229 push @newbusy, $clone;
232 if ( $failed_clone ) {
233 my $errstr = $failed_clone->errstr;
234 foreach my $clone (@newbusy, $failed_clone) {
235 $clone->pg_cancel if $clone->{pg_async_status} == 1;
241 if (my $statement = $phase->[0]) {
244 $clone = shift(@free);
245 } elsif ( !$MAX_HANDLES or
246 scalar(@free) + scalar(@busy) < $MAX_HANDLES ) {
247 $clone = $dbh->clone; # this will fail if over the server limit
251 my $rv = $clone->do($statement, {pg_async => PG_ASYNC});
254 shift @{ $phase }; # and actually take the statement off the queue
256 } # else I don't know, wait and retry
257 } # else too many handles, wait and retry
259 # all statements are dispatched
260 warn "Waiting for phase $i to complete\n";
263 } # while @$phase or @busy
266 warn "Schema changes complete.\n";
268 # warn "Pre-schema change upgrades completed in ". (time-$start). " seconds\n"; # if $DEBUG;
271 # dbdef->update_schema( dbdef_dist(datasrc), $dbh );
276 # normal case, run statements sequentially
279 foreach my $statement ( @statements ) {
281 $dbh->do( $statement )
282 or die "Error: ". $dbh->errstr. "\n executing: $statement";
286 warn "Schema upgrade completed in ". (time-$start). " seconds\n"; # if $DEBUG;
290 $hashref->{dry_run} = 1 if $DRY_RUN;
291 $hashref->{debug} = 1 if $DEBUG && $DRY_RUN;
292 prune_applications($hashref) unless $opt_s;
294 warn "Application pruning completed in ". (time-$start). " seconds\n"; # if $DEBUG;
297 print "\n" if $DRY_RUN;
299 if ( $dbh->{Driver}->{Name} =~ /^mysql/i && ! $opt_s ) {
301 foreach my $table (qw( svc_acct svc_phone cust_main_county )) {
303 my $sth = $dbh->prepare(
304 "SELECT COUNT(*) FROM duplicate_lock WHERE lockname = '$table'"
305 ) or die $dbh->errstr;
307 $sth->execute or die $sth->errstr;
309 unless ( $sth->fetchrow_arrayref->[0] ) {
311 $sth = $dbh->prepare(
312 "INSERT INTO duplicate_lock ( lockname ) VALUES ( '$table' )"
313 ) or die $dbh->errstr;
315 $sth->execute or die $sth->errstr;
321 warn "Duplication lock creation completed in ". (time-$start). " seconds\n"; # if $DEBUG;
326 $dbh->commit or die $dbh->errstr;
328 dbdef_create($dbh, $dbdef_file);
330 $dbh->disconnect or die $dbh->errstr;
332 delete $FS::Schema::dbdef_cache{$dbdef_file}; #force an actual reload
333 $FS::UID::AutoCommit = 0;
334 $FS::UID::callback_hack = 1;
335 $dbh = adminsuidsetup($user);
336 $FS::UID::callback_hack = 0;
337 unless ( $DRY_RUN || $opt_s ) {
338 my $dir = "%%%FREESIDE_CONF%%%/conf.". datasrc;
339 if (!scalar(qsearch('conf', {}))) {
340 my $error = FS::Conf::init_config($dir);
342 warn "CONFIGURATION UPGRADE FAILED\n";
343 $dbh->rollback or die $dbh->errstr;
348 $dbh->commit or die $dbh->errstr;
349 $dbh->disconnect or die $dbh->errstr;
351 $FS::UID::AutoCommit = 1;
353 $dbh = adminsuidsetup($user);
355 warn "Re-initialization with updated schema completed in ". (time-$start). " seconds\n"; # if $DEBUG;
358 #### NEW CUSTOM FIELDS:
359 # 3. migrate old virtual field data to the new custom fields
361 $cfsth = $dbh->prepare("SELECT * FROM virtual_field left join part_virtual_field using (vfieldpart)")
363 $cfsth->execute or die $cfsth->errstr;
365 while ( $cf = $cfsth->fetchrow_hashref ) {
366 my $tbl = $cf->{'dbtable'};
367 my $name = $cf->{'name'};
368 my $dtable = dbdef->table($tbl);
369 next unless $dtable && $dtable->primary_key; # XXX: warn first?
370 my $pkey = $dtable->primary_key;
371 next unless $dtable->column($pkey)->type =~ /int/i; # XXX: warn first?
372 push @cfst, "UPDATE $tbl set cf_$name = '".$cf->{'value'}."' WHERE $pkey = ".$cf->{'recnum'};
373 push @cfst, "DELETE FROM virtual_field WHERE vfieldnum = ".$cf->{'vfieldnum'};
375 foreach my $cfst ( @cfst ) {
378 or die "Error: ". $dbh->errstr. "\n executing: $cfst";
380 warn "Custom fields data upgrade completed";
382 upgrade_config(%upgrade_opts)
383 unless $DRY_RUN || $opt_s;
385 $dbh->commit or die $dbh->errstr;
387 warn "Config updates completed in ". (time-$start). " seconds\n"; # if $DEBUG;
390 upgrade(%upgrade_opts)
391 unless $DRY_RUN || $opt_s;
393 $dbh->commit or die $dbh->errstr;
395 warn "Table updates completed in ". (time-$start). " seconds\n"; # if $DEBUG;
398 upgrade_sqlradius(%upgrade_opts)
399 unless $DRY_RUN || $opt_s || $opt_r;
401 warn "SQL RADIUS updates completed in ". (time-$start). " seconds\n"; # if $DEBUG;
404 $dbh->commit or die $dbh->errstr;
405 $dbh->disconnect or die $dbh->errstr;
407 warn "Final commit and disconnection completed in ". (time-$start). " seconds; upgrade done!\n"; # if $DEBUG;
411 sub dbdef_create { # reverse engineer the schema from the DB and save to file
412 my( $dbh, $file ) = @_;
413 my $dbdef = new_native DBIx::DBSchema $dbh;
418 die "Usage:\n freeside-upgrade [ -d ] [ -q | -v ] [ -r ] [ -s ] [ -j ] [ -a ] user\n";
423 freeside-upgrade - Upgrades database schema for new freeside verisons.
427 freeside-upgrade [ -d ] [ -q | -v ] [ -r ] [ -s ] [ -j ] [ -a ]
431 Reads your existing database schema and updates it to match the current schema,
432 adding any columns or tables necessary.
434 Also performs other upgrade functions:
438 =item Calls FS:: Misc::prune::prune_applications (probably unnecessary every upgrade, but simply won't find any records to change)
440 =item If necessary, moves your configuration information from the filesystem in /usr/local/etc/freeside/conf.<datasrc> to the database.
444 [ -d ]: Dry run; output SQL statements (to STDOUT) only, but do not execute
447 [ -q ]: Run quietly. This may become the default at some point.
449 [ -v ]: Run verbosely, sending debugging information to STDERR. This is the
452 [ -s ]: Schema changes only. Useful for Pg/slony slaves where the data
453 changes will be replicated from the Pg/slony master.
455 [ -r ]: Skip sqlradius updates. Useful for occassions where the sqlradius
456 databases may be inaccessible.
458 [ -j ]: Run certain upgrades asychronously from the job queue. Recommended
459 for very large cust_main or part_pkg tables that take too long to
462 [ -a ]: Run schema changes in parallel (Pg only). DBIx::DBSchema minimum
463 version 0.41 recommended. Recommended only for large databases and
464 powerful database servers, to reduce upgrade time.