first pass RT4 merge, RT#13852

4 files changed, 396 insertions, 0 deletions

diff --git a/rt/t/fts/indexed_mysql.t b/rt/t/fts/indexed_mysql.t
new file mode 100644
index 000000000..8966f1cd4
--- /dev/null
+++ b/rt/t/fts/indexed_mysql.t
@@ -0,0 +1,134 @@
+#!/usr/bin/perl -w
+
+use strict;
+use warnings;
+
+use RT::Test tests => undef;
+plan skip_all => 'Not mysql' unless RT->Config->Get('DatabaseType') eq 'mysql';
+plan skip_all => "No SphinxSE in mysql" unless $RT::Handle->CheckSphinxSE;
+
+my %sphinx;
+$sphinx{'searchd'} = RT::Test->find_executable('searchd');
+$sphinx{'indexer'} = RT::Test->find_executable('indexer');
+
+plan skip_all => "No searchd and indexer under PATH"
+    unless $sphinx{'searchd'} && $sphinx{'indexer'};
+
+plan tests => 15;
+
+RT->Config->Set( FullTextSearch => Enable => 1, Indexed => 1, Table => 'AttachmentsIndex', MaxMatches => 1000 );
+
+setup_indexing();
+
+my $q = RT::Test->load_or_create_queue( Name => 'General' );
+ok $q && $q->id, 'loaded or created queue';
+my $queue = $q->Name;
+
+sub setup_indexing {
+    # Since we're not running a webserver in this test, use the
+    # known-safe port we determined at test setup
+    my $port = $RT::Test::port;
+    my ($exit_code, $output) = RT::Test->run_and_capture(
+        'no-ask'       => 1,
+        command        => $RT::SbinPath .'/rt-setup-fulltext-index',
+        dba            => $ENV{'RT_DBA_USER'},
+        'dba-password' => $ENV{'RT_DBA_PASSWORD'},
+        url            => "sphinx://localhost:$port/rt",
+    );
+    ok(!$exit_code, "setted up index");
+    diag "output: $output" if $ENV{'TEST_VERBOSE'};
+
+    my $tmp = $sphinx{'directory'} = File::Spec->catdir( RT::Test->temp_directory, 'sphinx' );
+    mkdir $tmp;
+
+    my $sphinx_conf = $output;
+    $sphinx_conf =~ s/.*?source rt {/source rt {/ms;
+    $sphinx_conf =~ s{\Q$RT::VarPath\E/sphinx/}{$tmp/}g;
+
+    $sphinx{'config'} = File::Spec->catfile( $tmp, 'sphinx.conf' );
+    {
+        open my $fh, ">", $sphinx{'config'};
+        print $fh $sphinx_conf;
+        close $fh;
+    }
+
+    sync_index();
+
+    {
+        my ($exit_code, $output) = RT::Test->run_and_capture(
+            command => $sphinx{'searchd'},
+            config => $sphinx{'config'},
+        );
+        ok(!$exit_code, "setted up index") or diag "output: $output";
+        $sphinx{'started'} = 1 if !$exit_code;
+    }
+}
+
+sub sync_index {
+    local $SIG{'CHLD'} = 'DEFAULT';
+    local $SIG{'PIPE'} = 'DEFAULT';
+    open my $fh, '-|',  $sphinx{'indexer'}, '--all',
+        '--config' => $sphinx{'config'},
+        $sphinx{'started'}? ('--rotate') : (),
+    ;
+    my $output = <$fh>;
+    close $fh;
+    my $exit_code = $?>>8;
+    ok(!$exit_code, "indexed") or diag "output: $output";
+
+    # We may need to wait a second for searchd to pick up the changes
+    sleep 1;
+}
+
+sub run_tests {
+    my @test = @_;
+    while ( my ($query, $checks) = splice @test, 0, 2 ) {
+        run_test( $query, %$checks );
+    }
+}
+
+my @tickets;
+sub run_test {
+    my ($query, %checks) = @_;
+    my $query_prefix = join ' OR ', map 'id = '. $_->id, @tickets;
+
+    my $tix = RT::Tickets->new(RT->SystemUser);
+    $tix->FromSQL( "( $query_prefix ) AND ( $query )" );
+
+    my $error = 0;
+
+    my $count = 0;
+    $count++ foreach grep $_, values %checks;
+    is($tix->Count, $count, "found correct number of ticket(s) by '$query'") or $error = 1;
+
+    my $good_tickets = ($tix->Count == $count);
+    while ( my $ticket = $tix->Next ) {
+        next if $checks{ $ticket->Subject };
+        diag $ticket->Subject ." ticket has been found when it's not expected";
+        $good_tickets = 0;
+    }
+    ok( $good_tickets, "all tickets are good with '$query'" ) or $error = 1;
+
+    diag "Wrong SQL query for '$query':". $tix->BuildSelectQuery if $error;
+}
+
+@tickets = RT::Test->create_tickets(
+    { Queue => $q->id },
+    { Subject => 'book', Content => 'book' },
+    { Subject => 'bar', Content => 'bar' },
+);
+sync_index();
+
+run_tests(
+    "Content LIKE 'book'" => { book => 1, bar => 0 },
+    "Content LIKE 'bar'" => { book => 0, bar => 1 },
+);
+
+END {
+    my $Test = RT::Test->builder;
+    return if $Test->{Original_Pid} != $$;
+    return unless $sphinx{'started'};
+
+    my $pid = int RT::Test->file_content([$sphinx{'directory'}, 'searchd.pid']);
+    kill TERM => $pid if $pid;
+}
diff --git a/rt/t/fts/indexed_oracle.t b/rt/t/fts/indexed_oracle.t
new file mode 100644
index 000000000..5d71712d0
--- /dev/null
+++ b/rt/t/fts/indexed_oracle.t
@@ -0,0 +1,82 @@
+#!/usr/bin/perl -w
+
+use strict;
+use warnings;
+
+use RT::Test tests => undef;
+plan skip_all => 'Not Oracle' unless RT->Config->Get('DatabaseType') eq 'Oracle';
+plan tests => 13;
+
+RT->Config->Set( FullTextSearch => Enable => 1, Indexed => 1 );
+
+setup_indexing();
+
+my $q = RT::Test->load_or_create_queue( Name => 'General' );
+ok $q && $q->id, 'loaded or created queue';
+my $queue = $q->Name;
+
+sub setup_indexing {
+    my %args = (
+        'no-ask'       => 1,
+        command        => $RT::SbinPath .'/rt-setup-fulltext-index',
+        dba            => $ENV{'RT_DBA_USER'},
+        'dba-password' => $ENV{'RT_DBA_PASSWORD'},
+    );
+    my ($exit_code, $output) = RT::Test->run_and_capture( %args );
+    ok(!$exit_code, "setted up index") or diag "output: $output";
+}
+
+sub sync_index {
+    my %args = (
+        command => $RT::SbinPath .'/rt-fulltext-indexer',
+    );
+    my ($exit_code, $output) = RT::Test->run_and_capture( %args );
+    ok(!$exit_code, "synced the index") or diag "output: $output";
+}
+
+sub run_tests {
+    my @test = @_;
+    while ( my ($query, $checks) = splice @test, 0, 2 ) {
+        run_test( $query, %$checks );
+    }
+}
+
+my @tickets;
+sub run_test {
+    my ($query, %checks) = @_;
+    my $query_prefix = join ' OR ', map 'id = '. $_->id, @tickets;
+
+    my $tix = RT::Tickets->new(RT->SystemUser);
+    $tix->FromSQL( "( $query_prefix ) AND ( $query )" );
+
+    my $error = 0;
+
+    my $count = 0;
+    $count++ foreach grep $_, values %checks;
+    is($tix->Count, $count, "found correct number of ticket(s) by '$query'") or $error = 1;
+
+    my $good_tickets = ($tix->Count == $count);
+    while ( my $ticket = $tix->Next ) {
+        next if $checks{ $ticket->Subject };
+        diag $ticket->Subject ." ticket has been found when it's not expected";
+        $good_tickets = 0;
+    }
+    ok( $good_tickets, "all tickets are good with '$query'" ) or $error = 1;
+
+    diag "Wrong SQL query for '$query':". $tix->BuildSelectQuery if $error;
+}
+
+@tickets = RT::Test->create_tickets(
+    { Queue => $q->id },
+    { Subject => 'book', Content => 'book' },
+    { Subject => 'bar', Content => 'bar' },
+);
+sync_index();
+
+run_tests(
+    "Content LIKE 'book'" => { book => 1, bar => 0 },
+    "Content LIKE 'bar'" => { book => 0, bar => 1 },
+);
+
+@tickets = ();
+
diff --git a/rt/t/fts/indexed_pg.t b/rt/t/fts/indexed_pg.t
new file mode 100644
index 000000000..c437c1f4f
--- /dev/null
+++ b/rt/t/fts/indexed_pg.t
@@ -0,0 +1,119 @@
+#!/usr/bin/perl -w
+
+use strict;
+use warnings;
+
+use RT::Test tests => undef;
+plan skip_all => 'Not Pg' unless RT->Config->Get('DatabaseType') eq 'Pg';
+
+my ($major, $minor) = $RT::Handle->dbh->get_info(18) =~ /^0*(\d+)\.0*(\d+)/;
+plan skip_all => "Need Pg 8.2 or higher; we have $major.$minor"
+    if "$major.$minor" < 8.2;
+
+plan tests => 36;
+
+RT->Config->Set( FullTextSearch => Enable => 1, Indexed => 1, Column => 'ContentIndex', Table => 'Attachments' );
+
+setup_indexing();
+
+my $q = RT::Test->load_or_create_queue( Name => 'General' );
+ok $q && $q->id, 'loaded or created queue';
+my $queue = $q->Name;
+
+sub setup_indexing {
+    my %args = (
+        'no-ask'       => 1,
+        command        => $RT::SbinPath .'/rt-setup-fulltext-index',
+        dba            => $ENV{'RT_DBA_USER'},
+        'dba-password' => $ENV{'RT_DBA_PASSWORD'},
+    );
+    my ($exit_code, $output) = RT::Test->run_and_capture( %args );
+    ok(!$exit_code, "setted up index") or diag "output: $output";
+}
+
+sub sync_index {
+    my %args = (
+        command => $RT::SbinPath .'/rt-fulltext-indexer',
+    );
+    my ($exit_code, $output) = RT::Test->run_and_capture( %args );
+    ok(!$exit_code, "setted up index") or diag "output: $output";
+}
+
+sub run_tests {
+    my @test = @_;
+    while ( my ($query, $checks) = splice @test, 0, 2 ) {
+        run_test( $query, %$checks );
+    }
+}
+
+my @tickets;
+sub run_test {
+    my ($query, %checks) = @_;
+    my $query_prefix = join ' OR ', map 'id = '. $_->id, @tickets;
+
+    my $tix = RT::Tickets->new(RT->SystemUser);
+    $tix->FromSQL( "( $query_prefix ) AND ( $query )" );
+
+    my $error = 0;
+
+    my $count = 0;
+    $count++ foreach grep $_, values %checks;
+    is($tix->Count, $count, "found correct number of ticket(s) by '$query'") or $error = 1;
+
+    my $good_tickets = ($tix->Count == $count);
+    while ( my $ticket = $tix->Next ) {
+        next if $checks{ $ticket->id };
+        diag $ticket->Subject ." ticket has been found when it's not expected";
+        $good_tickets = 0;
+    }
+    ok( $good_tickets, "all tickets are good with '$query'" ) or $error = 1;
+
+    diag "Wrong SQL query for '$query':". $tix->BuildSelectQuery if $error;
+}
+
+@tickets = RT::Test->create_tickets(
+    { Queue => $q->id },
+    { Subject => 'fts test 1', Content => 'book' },
+    { Subject => 'fts test 2', Content => 'bars'  },
+);
+sync_index();
+
+my $book = $tickets[0];
+my $bars = $tickets[1];
+
+run_tests(
+    "Content LIKE 'book'" => { $book->id => 1, $bars->id => 0 },
+    "Content LIKE 'bars'" => { $book->id => 0, $bars->id => 1 },
+
+    # make sure that Pg stemming works
+    "Content LIKE 'books'" => { $book->id => 1, $bars->id => 0 },
+    "Content LIKE 'bar'"   => { $book->id => 0, $bars->id => 1 },
+
+    # no matches
+    "Content LIKE 'baby'" => { $book->id => 0, $bars->id => 0 },
+    "Content LIKE 'pubs'" => { $book->id => 0, $bars->id => 0 },
+);
+
+# Test the "ts_vector too long" skip
+my $content = "";
+$content .= "$_\n" for 1..200_000;
+@tickets = RT::Test->create_tickets(
+    { Queue => $q->id },
+    { Subject => 'Short content', Content => '50' },
+    { Subject => 'Long content',  Content => $content  },
+    { Subject => 'More short',    Content => '50' },
+);
+
+my ($exit_code, $output) = RT::Test->run_and_capture(
+    command => $RT::SbinPath .'/rt-fulltext-indexer'
+);
+like($output, qr/string is too long for tsvector/, "Got a warning for the ticket");
+ok(!$exit_code, "set up index");
+
+# The long content is skipped entirely
+run_tests(
+    "Content LIKE '1'"  => { $tickets[0]->id => 0, $tickets[1]->id => 0, $tickets[2]->id => 0 },
+    "Content LIKE '50'" => { $tickets[0]->id => 1, $tickets[1]->id => 0, $tickets[2]->id => 1 },
+);
+
+@tickets = ();
diff --git a/rt/t/fts/not_indexed.t b/rt/t/fts/not_indexed.t
new file mode 100644
index 000000000..0a1abd081
--- /dev/null
+++ b/rt/t/fts/not_indexed.t
@@ -0,0 +1,61 @@
+#!/usr/bin/perl -w
+
+use strict;
+use warnings;
+
+use RT::Test tests => 20;
+
+RT->Config->Set( FullTextSearch => Enable => 1, Indexed => 0 );
+
+my $q = RT::Test->load_or_create_queue( Name => 'General' );
+ok $q && $q->id, 'loaded or created queue';
+my $queue = $q->Name;
+
+sub run_tests {
+    my @test = @_;
+    while ( my ($query, $checks) = splice @test, 0, 2 ) {
+        run_test( $query, %$checks );
+    }
+}
+
+my @tickets;
+sub run_test {
+    my ($query, %checks) = @_;
+    my $query_prefix = join ' OR ', map 'id = '. $_->id, @tickets;
+
+    my $tix = RT::Tickets->new(RT->SystemUser);
+    $tix->FromSQL( "( $query_prefix ) AND ( $query )" );
+
+    my $error = 0;
+
+    my $count = 0;
+    $count++ foreach grep $_, values %checks;
+    is($tix->Count, $count, "found correct number of ticket(s) by '$query'") or $error = 1;
+
+    my $good_tickets = ($tix->Count == $count);
+    while ( my $ticket = $tix->Next ) {
+        next if $checks{ $ticket->Subject };
+        diag $ticket->Subject ." ticket has been found when it's not expected";
+        $good_tickets = 0;
+    }
+    ok( $good_tickets, "all tickets are good with '$query'" ) or $error = 1;
+
+    diag "Wrong SQL query for '$query':". $tix->BuildSelectQuery if $error;
+}
+
+@tickets = RT::Test->create_tickets(
+    { Queue => $q->id },
+    { Subject => 'book', Content => 'book' },
+    { Subject => 'bar', Content => 'bar' },
+    { Subject => 'no content', Content => undef },
+);
+
+run_tests(
+    "Content LIKE 'book'" => { book => 1, bar => 0 },
+    "Content LIKE 'bar'" => { book => 0, bar => 1  },
+    "(Content LIKE 'baz' OR Subject LIKE 'con')" => { 'no content' => 1 },
+    "(Content LIKE 'bar' OR Subject LIKE 'con')" => { 'no content' => 1, bar => 1 },
+    "(Content LIKE 'bar' OR Subject LIKE 'missing')" => { bar => 1 },
+);
+
+