2 # BEGIN BPS TAGGED BLOCK {{{
6 # This software is Copyright (c) 1996-2017 Best Practical Solutions, LLC
7 # <sales@bestpractical.com>
9 # (Except where explicitly superseded by other copyright notices)
14 # This work is made available to you under the terms of Version 2 of
15 # the GNU General Public License. A copy of that license should have
16 # been provided with this software, but in any event can be snarfed
19 # This work is distributed in the hope that it will be useful, but
20 # WITHOUT ANY WARRANTY; without even the implied warranty of
21 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
22 # General Public License for more details.
24 # You should have received a copy of the GNU General Public License
25 # along with this program; if not, write to the Free Software
26 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
27 # 02110-1301 or visit their web page on the internet at
28 # http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
31 # CONTRIBUTION SUBMISSION POLICY:
33 # (The following paragraph is not intended to limit the rights granted
34 # to you to modify and distribute this software under the terms of
35 # the GNU General Public License and is only of importance to you if
36 # you choose to contribute your changes and enhancements to the
37 # community by submitting them to Best Practical Solutions, LLC.)
39 # By intentionally submitting any modifications, corrections or
40 # derivatives to this work, or any other work intended for use with
41 # Request Tracker, to Best Practical Solutions, LLC, you confirm that
42 # you are the copyright holder for those contributions and you grant
43 # Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable,
44 # royalty-free, perpetual, license to use, copy, create derivative
45 # works based on those contributions, and sublicense and distribute
46 # those contributions and any derivatives thereof.
48 # END BPS TAGGED BLOCK }}}
52 # fix lib paths, some may be relative
55 my @libs = ("@RT_LIB_PATH@", "@LOCAL_LIB_PATH@");
59 unless ( File::Spec->file_name_is_absolute($lib) ) {
61 if ( File::Spec->file_name_is_absolute(__FILE__) ) {
62 $bin_path = ( File::Spec->splitpath(__FILE__) )[1];
67 $bin_path = $FindBin::Bin;
70 $lib = File::Spec->catfile( $bin_path, File::Spec->updir, $lib );
81 @RT::Record::ISA = qw( DBIx::SearchBuilder::Record RT::Base );
84 use RT::Migrate::Serializer::File;
109 "hyperlink-unmigrated!",
116 ) or Pod::Usage::pod2usage();
118 Pod::Usage::pod2usage(-verbose => 1) if $OPT{help};
121 $args{Directory} = $OPT{directory};
122 $args{Force} = $OPT{force};
123 $args{MaxFileSize} = $OPT{size} if $OPT{size};
125 $args{AllUsers} = $OPT{users} if defined $OPT{users};
126 $args{AllGroups} = $OPT{groups} if defined $OPT{groups};
127 $args{FollowDeleted} = $OPT{deleted} if defined $OPT{deleted};
129 $args{FollowScrips} = $OPT{scrips} if defined $OPT{scrips};
130 $args{FollowTickets} = $OPT{tickets} if defined $OPT{tickets};
131 $args{FollowACL} = $OPT{acls} if defined $OPT{acls};
133 $args{HyperlinkUnmigrated} = $OPT{'hyperlink-unmigrated'} if defined $OPT{'hyperlink-unmigrated'};
135 $args{Clone} = $OPT{clone} if $OPT{clone};
136 $args{Incremental} = $OPT{incremental} if $OPT{incremental};
138 $args{GC} = defined $OPT{gc} ? $OPT{gc} : 5000;
139 $args{Page} = defined $OPT{page} ? $OPT{page} : 100;
141 if ($OPT{'limit-queues'}) {
144 for my $name (split ',', join ',', @{ $OPT{'limit-queues'} }) {
145 $name =~ s/^\s+//; $name =~ s/\s+$//;
146 my $queue = RT::Queue->new(RT->SystemUser);
149 die "Unable to load queue '$name'";
151 push @queue_ids, $queue->Id;
154 $args{Queues} = \@queue_ids;
157 if ($OPT{'limit-cfs'}) {
160 for my $name (split ',', join ',', @{ $OPT{'limit-cfs'} }) {
161 $name =~ s/^\s+//; $name =~ s/\s+$//;
164 if ($name =~ /^\d+$/) {
168 my $cfs = RT::CustomFields->new(RT->SystemUser);
169 $cfs->Limit(FIELD => 'Name', VALUE => $name);
171 die "Unable to load any custom field named '$name'";
173 push @cf_ids, map { $_->Id } @{ $cfs->ItemsArrayRef };
177 $args{CustomFields} = \@cf_ids;
180 if (($OPT{clone} or $OPT{incremental})
181 and grep { /^(users|groups|deleted|scrips|tickets|acls)$/ } keys %OPT) {
182 die "You cannot specify object types when cloning.\n\nPlease see $0 --help.\n";
187 my $gnuplot = `which gnuplot`;
189 if (-t STDOUT and not $OPT{verbose} and not $OPT{quiet}) {
190 $args{Progress} = RT::Migrate::progress(
192 bottom => sub { print "\n$msg"; $msg = ""; },
193 counts => sub { $walker->ObjectCount },
194 max => { estimate() },
196 $args{MessageHandler} = sub {
197 print "\r", " "x60, "\r", $_[-1]; $msg = $_[-1];
201 $args{Verbose} = 0 if $OPT{quiet};
204 $walker = RT::Migrate::Serializer::File->new( %args );
206 my $log = RT::Migrate::setup_logging( $walker->{Directory} => 'serializer.log' );
207 print "Logging warnings and errors to $log\n" if $log;
209 print "Beginning database serialization...";
210 my %counts = $walker->Export;
212 my @files = $walker->Files;
213 print "Wrote @{[scalar @files]} files:\n";
214 print " $_\n" for @files;
217 print "Total object counts:\n";
218 for (sort {$counts{$b} <=> $counts{$a}} keys %counts) {
219 printf "%8d %s\n", $counts{$_}, $_;
222 if ($log and -s $log) {
223 print STDERR "\n! Some warnings or errors occurred during serialization."
224 ."\n! Please see $log for details.\n\n";
233 # Expected types we'll serialize
234 my @types = map {"RT::$_"} qw/
235 Queue Ticket Transaction Attachment Link
236 User Group GroupMember Attribute
237 CustomField CustomFieldValue
238 ObjectCustomField ObjectCustomFieldValue
241 for my $class (@types) {
242 print "Estimating $class count...";
243 my $collection = $class . "s";
244 if ($collection->require) {
245 my $objs = $collection->new( RT->SystemUser );
248 $objs->{allow_deleted_search} = 1 if $class eq "RT::Ticket";
249 $e{$class} = $objs->DBIx::SearchBuilder::Count;
251 print "\r", " "x60, "\r";
259 my ($elapsed, $rows, $cols) = @_;
260 my $length = $walker->StackSize;
261 my $file = $walker->Directory . "/progress.plot";
262 open(my $dat, ">>", $file);
263 printf $dat "%10.3f\t%8d\n", $elapsed, $length;
266 if ($rows <= 24 or not $gnuplot) {
271 set term dumb $cols @{[$rows - 12]};
272 set xlabel "Seconds";
276 set title "Queue length";
277 plot "$file" using 1:2 with lines
280 if ($? == 0 and $gnuplot) {
281 $gnuplot =~ s/^(\s*\n)//;
285 warn "Couldn't run gnuplot (\$? == $?): $!\n";
288 print "\n" for 1..($rows - 13);
294 rt-serializer - Serialize an RT database to disk
298 rt-validator --check && rt-serializer
300 This script is used to write out the entire RT database to disk, for
301 later import into a different RT instance. It requires that the data in
302 the database be self-consistent, in order to do so; please make sure
303 that the database being exported passes validation by L<rt-validator>
304 before attempting to use C<rt-serializer>.
306 While running, it will attempt to estimate the number of remaining
307 objects to be serialized; these estimates are pessimistic, and will be
308 incorrect if C<--no-users>, C<--no-groups>, or C<--no-tickets> are used.
310 If the controlling terminal is large enough (more than 25 columns high)
311 and the C<gnuplot> program is installed, it will also show a textual
312 graph of the queue size over time.
318 =item B<--directory> I<name>
320 The name of the output directory to write data files to, which should
321 not exist yet; it is a fatal error if it does. Defaults to
322 C<< ./I<$Organization>:I<Date>/ >>, where I<$Organization> is as set in
323 F<RT_SiteConfig.pm>, and I<Date> is today's date.
327 Remove the output directory before starting.
329 =item B<--size> I<megabytes>
331 By default, C<rt-serializer> chunks its output into data files which are
332 around 32Mb in size; this option is used to set a different threshold
333 size, in megabytes. Note that this is the threshold after which it
334 rotates to writing a new file, and is as such the I<lower bound> on the
335 size of each output file.
339 By default, all privileged users are serialized; passing C<--no-users>
340 limits it to only those users which are referenced by serialized tickets
341 and history, and are thus necessary for internal consistency.
345 By default, all groups are serialized; passing C<--no-groups> limits it
346 to only system-internal groups, which are needed for internal
349 =item B<--no-deleted>
351 By default, all tickets, including deleted tickets, are serialized;
352 passing C<--no-deleted> skips deleted tickets during serialization.
356 No scrips or templates are serialized by default; this option forces all
357 scrips and templates to be serialized.
361 No ACLs are serialized by default; this option forces all ACLs to be
364 =item B<--no-tickets>
366 Skip serialization of all ticket data.
368 =item B<--limit-queues>
370 Takes a list of queue IDs or names separated by commas. When provided, only
371 that set of queues (and the tickets in them) will be serialized.
375 Takes a list of custom field IDs or names separated by commas. When provided,
376 only that set of custom fields will be serialized.
378 =item B<--hyperlink-unmigrated>
380 Replace links to local records which are not being migrated with hyperlinks.
381 The hyperlinks will use the serializing RT's configured URL.
383 Without this option, such links are instead dropped, and transactions which
384 had updated such links will be replaced with an explanatory message.
388 Serializes your entire database, creating a clone. This option should
389 be used if you want to migrate your RT database from one database type
390 to another (e.g. MySQL to Postgres). It is an error to combine
391 C<--clone> with any option that limits object types serialized. No
392 dependency walking is performed when cloning. C<rt-importer> will detect
393 that your serialized data set was generated by a clone.
395 =item B<--incremental>
397 Will generate an incremenal serialized dataset using the data stored in
398 your IncrementalRecords database table. This assumes that you have created
399 that table and run RT using the Record_Local.pm shim as documented in
400 C<docs/incremental-export/>.
404 Adjust how often the garbage collection sweep is done; lower numbers are
405 more frequent. See L</GARBAGE COLLECTION>.
409 Adjust how many rows are pulled from the database in a single query. Disable
410 paging by setting this to 0. Defaults to 100.
412 Keep in mind that rows from RT's Attachments table are the limiting factor when
413 determining page size. You should likely be aiming for 60-75% of your total
414 memory on an otherwise unloaded box.
418 Do not show graphical progress UI.
422 Do not show graphical progress UI, but rather log was each row is
427 =head1 GARBAGE COLLECTION
429 C<rt-serializer> maintains a priority queue of objects to serialize, or
430 searches which may result in objects to serialize. When inserting into
431 this queue, it does no checking if the object in question is already in
432 the queue, or if the search will contain any results. These checks are
433 done when the object reaches the front of the queue, or during periodic
436 During periodic garbage collection, the entire queue is swept for
437 objects which have already been serialized, occur more than once in the
438 queue, and searches which contain no results in the database. This is
439 done to reduce the memory footprint of the serialization process, and is
440 triggered when enough new objects have been placed in the queue. This
441 parameter is tunable via the C<--gc> parameter, which defaults to
442 running garbage collection every 5,000 objects inserted into the queue;
443 smaller numbers will result in more frequent garbage collection.
445 The default of 5,000 is roughly tuned based on a database with several
446 thousand tickets, but optimal values will vary wildly depending on
447 database configuration and size. Values as low as 25 have provided
448 speedups with smaller databases; if speed is a factor, experimenting
449 with different C<--gc> values may be helpful. Note that there are
450 significant boundary condition changes in serialization rate, as the
451 queue empties and fills, causing the time estimates to be rather
452 imprecise near the start and end of the process.
454 Setting C<--gc> to 0 turns off all garbage collection. Be aware that
455 this will bloat the memory usage of the serializer. Any negative value
456 for C<--gc> turns off periodic garbage collection and instead objects
457 already serialized or in the queue are checked for at the time they