2 # BEGIN BPS TAGGED BLOCK {{{
6 # This software is Copyright (c) 1996-2015 Best Practical Solutions, LLC
7 # <sales@bestpractical.com>
9 # (Except where explicitly superseded by other copyright notices)
14 # This work is made available to you under the terms of Version 2 of
15 # the GNU General Public License. A copy of that license should have
16 # been provided with this software, but in any event can be snarfed
19 # This work is distributed in the hope that it will be useful, but
20 # WITHOUT ANY WARRANTY; without even the implied warranty of
21 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
22 # General Public License for more details.
24 # You should have received a copy of the GNU General Public License
25 # along with this program; if not, write to the Free Software
26 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
27 # 02110-1301 or visit their web page on the internet at
28 # http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
31 # CONTRIBUTION SUBMISSION POLICY:
33 # (The following paragraph is not intended to limit the rights granted
34 # to you to modify and distribute this software under the terms of
35 # the GNU General Public License and is only of importance to you if
36 # you choose to contribute your changes and enhancements to the
37 # community by submitting them to Best Practical Solutions, LLC.)
39 # By intentionally submitting any modifications, corrections or
40 # derivatives to this work, or any other work intended for use with
41 # Request Tracker, to Best Practical Solutions, LLC, you confirm that
42 # you are the copyright holder for those contributions and you grant
43 # Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable,
44 # royalty-free, perpetual, license to use, copy, create derivative
45 # works based on those contributions, and sublicense and distribute
46 # those contributions and any derivatives thereof.
48 # END BPS TAGGED BLOCK }}}
52 # fix lib paths, some may be relative
55 my @libs = ("@RT_LIB_PATH@", "@LOCAL_LIB_PATH@");
59 unless ( File::Spec->file_name_is_absolute($lib) ) {
61 if ( File::Spec->file_name_is_absolute(__FILE__) ) {
62 $bin_path = ( File::Spec->splitpath(__FILE__) )[1];
67 $bin_path = $FindBin::Bin;
70 $lib = File::Spec->catfile( $bin_path, File::Spec->updir, $lib );
81 @RT::Record::ISA = qw( DBIx::SearchBuilder::Record RT::Base );
84 use RT::Migrate::Serializer::File;
113 ) or Pod::Usage::pod2usage();
115 Pod::Usage::pod2usage(-verbose => 1) if $OPT{help};
118 $args{Directory} = $OPT{directory};
119 $args{Force} = $OPT{force};
120 $args{MaxFileSize} = $OPT{size} if $OPT{size};
122 $args{AllUsers} = $OPT{users} if defined $OPT{users};
123 $args{AllGroups} = $OPT{groups} if defined $OPT{groups};
124 $args{FollowDeleted} = $OPT{deleted} if defined $OPT{deleted};
126 $args{FollowScrips} = $OPT{scrips} if defined $OPT{scrips};
127 $args{FollowTickets} = $OPT{tickets} if defined $OPT{tickets};
128 $args{FollowACL} = $OPT{acls} if defined $OPT{acls};
130 $args{Clone} = $OPT{clone} if $OPT{clone};
131 $args{Incremental} = $OPT{incremental} if $OPT{incremental};
133 $args{GC} = defined $OPT{gc} ? $OPT{gc} : 5000;
134 $args{Page} = defined $OPT{page} ? $OPT{page} : 100;
136 if (($OPT{clone} or $OPT{incremental})
137 and grep { /^(users|groups|deleted|scrips|tickets|acls)$/ } keys %OPT) {
138 die "You cannot specify object types when cloning.\n\nPlease see $0 --help.\n";
143 my $gnuplot = `which gnuplot`;
145 if (-t STDOUT and not $OPT{verbose} and not $OPT{quiet}) {
146 $args{Progress} = RT::Migrate::progress(
148 bottom => sub { print "\n$msg"; $msg = ""; },
149 counts => sub { $walker->ObjectCount },
150 max => { estimate() },
152 $args{MessageHandler} = sub {
153 print "\r", " "x60, "\r", $_[-1]; $msg = $_[-1];
157 $args{Verbose} = 0 if $OPT{quiet};
160 $walker = RT::Migrate::Serializer::File->new( %args );
162 my $log = RT::Migrate::setup_logging( $walker->{Directory} => 'serializer.log' );
163 print "Logging warnings and errors to $log\n" if $log;
165 print "Beginning database serialization...";
166 my %counts = $walker->Export;
168 my @files = $walker->Files;
169 print "Wrote @{[scalar @files]} files:\n";
170 print " $_\n" for @files;
173 print "Total object counts:\n";
174 for (sort {$counts{$b} <=> $counts{$a}} keys %counts) {
175 printf "%8d %s\n", $counts{$_}, $_;
178 if ($log and -s $log) {
179 print STDERR "\n! Some warnings or errors occurred during serialization."
180 ."\n! Please see $log for details.\n\n";
189 # Expected types we'll serialize
190 my @types = map {"RT::$_"} qw/
191 Queue Ticket Transaction Attachment Link
192 User Group GroupMember Attribute
193 CustomField CustomFieldValue
194 ObjectCustomField ObjectCustomFieldValue
197 for my $class (@types) {
198 print "Estimating $class count...";
199 my $collection = $class . "s";
200 if ($collection->require) {
201 my $objs = $collection->new( RT->SystemUser );
204 $objs->{allow_deleted_search} = 1 if $class eq "RT::Ticket";
205 $e{$class} = $objs->DBIx::SearchBuilder::Count;
207 print "\r", " "x60, "\r";
215 my ($elapsed, $rows, $cols) = @_;
216 my $length = $walker->StackSize;
217 my $file = $walker->Directory . "/progress.plot";
218 open(my $dat, ">>", $file);
219 printf $dat "%10.3f\t%8d\n", $elapsed, $length;
222 if ($rows <= 24 or not $gnuplot) {
227 set term dumb $cols @{[$rows - 12]};
228 set xlabel "Seconds";
232 set title "Queue length";
233 plot "$file" using 1:2 with lines
236 if ($? == 0 and $gnuplot) {
237 $gnuplot =~ s/^(\s*\n)//;
241 warn "Couldn't run gnuplot (\$? == $?): $!\n";
244 print "\n" for 1..($rows - 13);
250 rt-serializer - Serialize an RT database to disk
254 rt-validator --check && rt-serializer
256 This script is used to write out the entire RT database to disk, for
257 later import into a different RT instance. It requires that the data in
258 the database be self-consistent, in order to do so; please make sure
259 that the database being exported passes validation by L<rt-validator>
260 before attempting to use C<rt-serializer>.
262 While running, it will attempt to estimate the number of remaining
263 objects to be serialized; these estimates are pessimistic, and will be
264 incorrect if C<--no-users>, C<--no-groups>, or C<--no-tickets> are used.
266 If the controlling terminal is large enough (more than 25 columns high)
267 and the C<gnuplot> program is installed, it will also show a textual
268 graph of the queue size over time.
274 =item B<--directory> I<name>
276 The name of the output directory to write data files to, which should
277 not exist yet; it is a fatal error if it does. Defaults to
278 C<< ./I<$Organization>:I<Date>/ >>, where I<$Organization> is as set in
279 F<RT_SiteConfig.pm>, and I<Date> is today's date.
283 Remove the output directory before starting.
285 =item B<--size> I<megabytes>
287 By default, C<rt-serializer> chunks its output into data files which are
288 around 32Mb in size; this option is used to set a different threshold
289 size, in megabytes. Note that this is the threshold after which it
290 rotates to writing a new file, and is as such the I<lower bound> on the
291 size of each output file.
295 By default, all privileged users are serialized; passing C<--no-users>
296 limits it to only those users which are referenced by serialized tickets
297 and history, and are thus necessary for internal consistency.
301 By default, all groups are serialized; passing C<--no-groups> limits it
302 to only system-internal groups, which are needed for internal
305 =item B<--no-deleted>
307 By default, all tickets, including deleted tickets, are serialized;
308 passing C<--no-deleted> skips deleted tickets during serialization.
312 No scrips or templates are serialized by default; this option forces all
313 scrips and templates to be serialized.
317 No ACLs are serialized by default; this option forces all ACLs to be
320 =item B<--no-tickets>
322 Skip serialization of all ticket data.
326 Serializes your entire database, creating a clone. This option should
327 be used if you want to migrate your RT database from one database type
328 to another (e.g. MySQL to Postgres). It is an error to combine
329 C<--clone> with any option that limits object types serialized. No
330 dependency walking is performed when cloning. C<rt-importer> will detect
331 that your serialized data set was generated by a clone.
333 =item B<--incremental>
335 Will generate an incremenal serialized dataset using the data stored in
336 your IncrementalRecords database table. This assumes that you have created
337 that table and run RT using the Record_Local.pm shim as documented in
338 C<docs/incremental-export/>.
342 Adjust how often the garbage collection sweep is done; lower numbers are
343 more frequent. See L</GARBAGE COLLECTION>.
347 Adjust how many rows are pulled from the database in a single query. Disable
348 paging by setting this to 0. Defaults to 100.
350 Keep in mind that rows from RT's Attachments table are the limiting factor when
351 determining page size. You should likely be aiming for 60-75% of your total
352 memory on an otherwise unloaded box.
356 Do not show graphical progress UI.
360 Do not show graphical progress UI, but rather log was each row is
365 =head1 GARBAGE COLLECTION
367 C<rt-serializer> maintains a priority queue of objects to serialize, or
368 searches which may result in objects to serialize. When inserting into
369 this queue, it does no checking if the object in question is already in
370 the queue, or if the search will contain any results. These checks are
371 done when the object reaches the front of the queue, or during periodic
374 During periodic garbage collection, the entire queue is swept for
375 objects which have already been serialized, occur more than once in the
376 queue, and searches which contain no results in the database. This is
377 done to reduce the memory footprint of the serialization process, and is
378 triggered when enough new objects have been placed in the queue. This
379 parameter is tunable via the C<--gc> parameter, which defaults to
380 running garbage collection every 5,000 objects inserted into the queue;
381 smaller numbers will result in more frequent garbage collection.
383 The default of 5,000 is roughly tuned based on a database with several
384 thousand tickets, but optimal values will vary wildly depending on
385 database configuration and size. Values as low as 25 have provided
386 speedups with smaller databases; if speed is a factor, experimenting
387 with different C<--gc> values may be helpful. Note that there are
388 significant boundary condition changes in serialization rate, as the
389 queue empties and fills, causing the time estimates to be rather
390 imprecise near the start and end of the process.
392 Setting C<--gc> to 0 turns off all garbage collection. Be aware that
393 this will bloat the memory usage of the serializer. Any negative value
394 for C<--gc> turns off periodic garbage collection and instead objects
395 already serialized or in the queue are checked for at the time they