2 # BEGIN BPS TAGGED BLOCK {{{
6 # This software is Copyright (c) 1996-2012 Best Practical Solutions, LLC
7 # <sales@bestpractical.com>
9 # (Except where explicitly superseded by other copyright notices)
14 # This work is made available to you under the terms of Version 2 of
15 # the GNU General Public License. A copy of that license should have
16 # been provided with this software, but in any event can be snarfed
19 # This work is distributed in the hope that it will be useful, but
20 # WITHOUT ANY WARRANTY; without even the implied warranty of
21 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
22 # General Public License for more details.
24 # You should have received a copy of the GNU General Public License
25 # along with this program; if not, write to the Free Software
26 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
27 # 02110-1301 or visit their web page on the internet at
28 # http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
31 # CONTRIBUTION SUBMISSION POLICY:
33 # (The following paragraph is not intended to limit the rights granted
34 # to you to modify and distribute this software under the terms of
35 # the GNU General Public License and is only of importance to you if
36 # you choose to contribute your changes and enhancements to the
37 # community by submitting them to Best Practical Solutions, LLC.)
39 # By intentionally submitting any modifications, corrections or
40 # derivatives to this work, or any other work intended for use with
41 # Request Tracker, to Best Practical Solutions, LLC, you confirm that
42 # you are the copyright holder for those contributions and you grant
43 # Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable,
44 # royalty-free, perpetual, license to use, copy, create derivative
45 # works based on those contributions, and sublicense and distribute
46 # those contributions and any derivatives thereof.
48 # END BPS TAGGED BLOCK }}}
53 # fix lib paths, some may be relative
56 my @libs = ("/opt/rt3/lib", "/opt/rt3/local/lib");
60 unless ( File::Spec->file_name_is_absolute($lib) ) {
62 if ( File::Spec->file_name_is_absolute(__FILE__) ) {
63 $bin_path = ( File::Spec->splitpath(__FILE__) )[1];
68 $bin_path = $FindBin::Bin;
71 $lib = File::Spec->catfile( $bin_path, File::Spec->updir, $lib );
82 use RT::Interface::CLI ();
88 my @OPT_LIST = qw(help|h! debug!);
90 my $db_type = RT->Config->Get('DatabaseType');
91 if ( $db_type eq 'Pg' ) {
97 push @OPT_LIST, 'limit=i', 'all!';
99 elsif ( $db_type eq 'mysql' ) {
106 push @OPT_LIST, 'limit=i', 'all!', 'xmlpipe2!';
108 elsif ( $db_type eq 'Oracle' ) {
113 push @OPT_LIST, qw(memory=s);
116 use Getopt::Long qw(GetOptions);
117 GetOptions( \%OPT, @OPT_LIST );
119 if ( $OPT{'help'} ) {
120 RT::Interface::CLI->ShowHelp(
121 Sections => 'NAME|DESCRIPTION|'. uc($db_type),
125 my $fts_config = RT->Config->Get('FullTextSearch') || {};
126 unless ( $fts_config->{'Enable'} ) {
129 Full text search is disabled in your RT configuration. Run
130 /opt/rt3/sbin/rt-setup-fulltext-index to configure and enable it.
135 unless ( $fts_config->{'Indexed'} ) {
138 Full text search is enabled in your RT configuration, but not with any
139 full-text database indexing -- hence this tool is not required. Read
140 the documentation for %FullTextSearch in your RT_Config for more details.
146 if ( $db_type eq 'Oracle' ) {
147 my $index = $fts_config->{'IndexName'} || 'rt_fts_index';
148 $RT::Handle->dbh->do(
149 "begin ctx_ddl.sync_index(?, ?); end;", undef,
150 $index, $OPT{'memory'}
153 } elsif ( $db_type eq 'mysql' ) {
154 unless ($OPT{'xmlpipe2'}) {
157 Updates to the external Sphinx index are done via running the sphinx
167 my @types = qw(text html);
168 foreach my $type ( @types ) {
170 my $attachments = attachments($type);
174 VALUE => last_indexed($type)
176 $attachments->OrderBy( FIELD => 'id', ORDER => 'asc' );
177 $attachments->RowsPerPage( $OPT{'limit'} || 100 );
180 while ( my $a = $attachments->Next ) {
181 next if filter( $type, $a );
182 debug("Found attachment #". $a->id );
183 my $txt = extract($type, $a) or next;
185 process( $type, $a, $txt );
186 debug("Processed attachment #". $a->id );
188 finalize( $type, $attachments ) if $found;
190 goto REDO if $OPT{'all'} and $attachments->Count == ($OPT{'limit'} || 100)
195 my $res = RT::Attachments->new( RT->SystemUser );
196 my $txn_alias = $res->Join(
198 FIELD1 => 'TransactionId',
199 TABLE2 => 'Transactions',
204 FIELD => 'ObjectType',
205 VALUE => 'RT::Ticket',
207 my $ticket_alias = $res->Join(
208 ALIAS1 => $txn_alias,
209 FIELD1 => 'ObjectId',
214 ALIAS => $ticket_alias,
220 # On newer DBIx::SearchBuilder's, indicate that making the query DISTINCT
221 # is unnecessary because the joins won't produce duplicates. This
222 # drastically improves performance when fetching attachments.
223 $res->{joins_are_distinct} = 1;
225 return goto_specific(
227 error => "Don't know how to find $type attachments",
234 return goto_specific(
236 error => "Don't know how to find last indexed $type attachment for $db_type DB",
243 return goto_specific(
251 return goto_specific(
253 error => "No way to convert $type attachment into text",
259 return goto_specific(
261 error => "No processer for $db_type DB",
267 return goto_specific(
274 return goto_specific(
281 sub last_indexed_mysql {
283 my $attr = $RT::System->FirstAttribute('LastIndexedAttachments');
284 return 0 unless $attr;
285 return 0 unless exists $attr->{ $type };
286 return $attr->{ $type } || 0;
290 my ($type, $attachment, $text) = (@_);
292 my $doc = sphinx_template();
294 my $element = $doc->createElement('sphinx:document');
295 $element->setAttribute( id => $attachment->id );
296 $element->appendTextChild( content => $$text );
298 $doc->documentElement->appendChild( $element );
302 sub sphinx_template {
306 $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
307 my $root = $doc->createElement('sphinx:docset');
308 $doc->setDocumentElement( $root );
310 my $schema = $doc->createElement('sphinx:schema');
311 $root->appendChild( $schema );
312 foreach ( qw(content) ) {
313 my $field = $doc->createElement('sphinx:field');
314 $field->setAttribute( name => $_ );
315 $schema->appendChild( $field );
322 my ($type, $attachments) = @_;
323 sphinx_template()->toFH(*STDOUT, 1);
332 sub last_indexed_pg {
334 my $attachments = attachments( $type );
336 if ( $fts_config->{'Table'} && $fts_config->{'Table'} ne 'Attachments' ) {
337 $alias = $attachments->Join(
340 TABLE2 => $fts_config->{'Table'},
346 FIELD => $fts_config->{'Column'},
347 OPERATOR => 'IS NOT',
350 $attachments->OrderBy( FIELD => 'id', ORDER => 'desc' );
351 $attachments->RowsPerPage( 1 );
352 my $res = $attachments->First;
353 return 0 unless $res;
358 my ($type, $attachment, $text) = (@_);
360 my $dbh = $RT::Handle->dbh;
361 my $table = $fts_config->{'Table'};
362 my $column = $fts_config->{'Column'};
366 if ( my ($id) = $dbh->selectrow_array("SELECT id FROM $table WHERE id = ?", undef, $attachment->id) ) {
367 $query = "UPDATE $table SET $column = to_tsvector(?) WHERE id = ?";
369 $query = "INSERT INTO $table($column, id) VALUES(to_tsvector(?), ?)";
372 $query = "UPDATE Attachments SET $column = to_tsvector(?) WHERE id = ?";
375 my $status = eval { $dbh->do( $query, undef, $$text, $attachment->id ) };
377 if ($dbh->errstr =~ /string is too long for tsvector/) {
378 warn "Attachment @{[$attachment->id]} not indexed, as it contains too many unique words to be indexed";
380 die "error: ". $dbh->errstr;
385 sub attachments_text {
387 $res->Limit( FIELD => 'ContentType', VALUE => 'text/plain' );
392 my $attachment = shift;
393 my $text = $attachment->Content;
394 return undef unless defined $text && length($text);
398 sub attachments_html {
400 $res->Limit( FIELD => 'ContentType', VALUE => 'text/html' );
405 my $attachment = shift;
406 if ( my $parent = $attachment->ParentObj ) {
407 # skip html parts that are alternatives
408 return 1 if $parent->id
409 && $parent->ContentType eq 'mulitpart/alternative';
415 my $attachment = shift;
416 my $text = $attachment->Content;
417 return undef unless defined $text && length($text);
425 my $func = (caller(1))[3];
427 my $call = $func ."_". lc $args{'suffix'};
428 unless ( defined &$call ) {
429 return undef unless $args{'error'};
430 require Carp; Carp::croak( $args{'error'} );
432 @_ = @{ $args{'arguments'} };
438 sub debug { print @_, "\n" if $OPT{debug}; 1 }
439 sub error { $RT::Logger->error(_(@_)); 1 }
440 sub warning { $RT::Logger->warn(_(@_)); 1 }
444 rt-fulltext-indexer - Indexer for full text search
448 This is a helper script to keep full text indexes in sync with data.
449 Read F<docs/full_text_indexing.pod> for complete details on how and when
454 Ruslan Zakirov E<lt>ruz@bestpractical.comE<gt>,
455 Alex Vandiver E<lt>alexmv@bestpractical.comE<gt>