RT#18834: Cacti integration [database storage]

4 files changed, 322 insertions, 110 deletions

diff --git a/FS/FS/Cron/cacti_cleanup.pm b/FS/FS/Cron/cacti_cleanup.pm
new file mode 100644
index 0000000..f862627
--- /dev/null
+++ b/FS/FS/Cron/cacti_cleanup.pm
@@ -0,0 +1,19 @@
+package FS::Cron::cacti_cleanup;
+use base 'Exporter';
+use vars '@EXPORT_OK';
+
+use FS::Record qw( qsearch );
+use Data::Dumper;
+
+@EXPORT_OK = qw( cacti_cleanup );
+
+sub cacti_cleanup {
+  foreach my $export (qsearch({
+    'table' => 'part_export',
+    'hashref' => { 'exporttype' => 'cacti' }
+  })) {
+    $export->cleanup;
+  }
+}
+
+1;
diff --git a/FS/FS/Schema.pm b/FS/FS/Schema.pm
index 4bc3598..839a971 100644
--- a/FS/FS/Schema.pm
+++ b/FS/FS/Schema.pm
@@ -203,6 +203,7 @@ sub dbdef_dist {
            && ! /^legacy_cust_history$/
            && ( ! /^queue(_arg|_depend|_stat)?$/ || ! $opt->{'queue-no_history'} )
            && ! $tables_hashref_torrus->{$_}
+           && ! /^cacti_graph$/
          }
       $dbdef->tables
   ) {
@@ -7007,9 +7008,29 @@ sub tables_hashref {
                         ],
     },
 
-
-
-
+    'cacti_page' => {
+      'columns' => [
+        'cacti_pagenum',  'serial',   '',     '', '', '',
+        'exportnum',      'int',      'NULL', '', '', '',
+        'svcnum',         'int',      'NULL', '', '', '', 
+        'graphnum',       'int',      'NULL', '', '', '', 
+        'imported',       @date_type,             '', '',
+        'content',        'text',     'NULL', '', '', '',
+      ],
+      'primary_key' => 'cacti_pagenum',
+      'unique'  => [ ],
+      'index'   => [ ['svcnum'], ['imported'] ],
+      'foreign_keys' => [
+                          { columns    => [ 'svcnum' ],
+                            table      => 'cust_svc',
+                            references => [ 'svcnum' ],
+                          },
+                          { columns    => [ 'exportnum' ],
+                            table      => 'part_export',
+                            references => [ 'exportnum' ],
+                          },
+                        ],
+    },
 
     # name type nullability length default local
 
diff --git a/FS/FS/cacti_page.pm b/FS/FS/cacti_page.pm
new file mode 100644
index 0000000..febcae4
--- /dev/null
+++ b/FS/FS/cacti_page.pm
@@ -0,0 +1,135 @@
+package FS::cacti_page;
+use base qw( FS::Record );
+
+use strict;
+use FS::Record qw( qsearch qsearchs );
+
+=head1 NAME
+
+FS::cacti_page - Object methods for cacti_page records
+
+=head1 SYNOPSIS
+
+  use FS::cacti_page;
+
+  $record = new FS::cacti_page \%hash;
+  $record = new FS::table_name {
+              'exportnum' => 3,           #part_export associated with this page
+              'svcnum'   => 123,          #svc_broadband associated with this page
+              'graphnum' => 45,           #blank for svcnum index
+              'imported' => 1428358699,   #date of import
+              'content'  => $htmlcontent, #html containing base64-encoded images
+  };
+
+  $error = $record->insert;
+
+  $error = $new_record->replace($old_record);
+
+  $error = $record->delete;
+
+  $error = $record->check;
+
+=head1 DESCRIPTION
+
+An FS::cacti_page object represents an html page for viewing cacti graphs.
+FS::cacti_page inherits from FS::Record.  The following fields are currently supported:
+
+=over 4
+
+=item cacti_pagenum - primary key
+
+=item exportnum - part_export exportnum for this page
+
+=item svcnum - svc_broadband svcnum for this page
+
+=item graphnum - cacti graphnum for this page (blank for overview page)
+
+=item imported - date this page was imported
+
+=item content - text/html content of page, should not include newlines
+
+=back
+
+=head1 METHODS
+
+=over 4
+
+=item new HASHREF
+
+Creates a new object.  To add the object to the database, see L<"insert">.
+
+Note that this stores the hash reference, not a distinct copy of the hash it
+points to.  You can ask the object for a copy with the I<hash> method.
+
+=cut
+
+# the new method can be inherited from FS::Record, if a table method is defined
+
+sub table { 'cacti_page'; }
+
+=item insert
+
+Adds this record to the database.  If there is an error, returns the error,
+otherwise returns false.
+
+=cut
+
+# the insert method can be inherited from FS::Record
+
+=item delete
+
+Delete this record from the database.
+
+=cut
+
+# the delete method can be inherited from FS::Record
+
+=item replace OLD_RECORD
+
+Replaces the OLD_RECORD with this one in the database.  If there is an error,
+returns the error, otherwise returns false.
+
+=cut
+
+# the replace method can be inherited from FS::Record
+
+=item check
+
+Checks all fields to make sure this is a valid example.  If there is
+an error, returns the error, otherwise returns false.  Called by the insert
+and replace methods.
+
+=cut
+
+# the check method should currently be supplied - FS::Record contains some
+# data checking routines
+
+sub check {
+  my $self = shift;
+
+  my $error = 
+    $self->ut_numbern('cacti_pagenum', 'graphnum')
+    || $self->ut_foreign_key('exportnum','part_export','exportnum')
+    || $self->ut_foreign_key('svcnum','cust_svc','svcnum')
+    || $self->ut_number('imported')
+    || $self->ut_text('content')
+  ;
+  return $error if $error;
+
+  $self->SUPER::check;
+}
+
+=back
+
+=head1 BUGS
+
+Will be described here once found.
+
+=head1 SEE ALSO
+
+L<FS::Record>
+
+=cut
+
+1;
+
diff --git a/FS/FS/part_export/cacti.pm b/FS/FS/part_export/cacti.pm
index 1f5f64c..abeb5e4 100644
--- a/FS/FS/part_export/cacti.pm
+++ b/FS/FS/part_export/cacti.pm
@@ -1,13 +1,31 @@
 package FS::part_export::cacti;
 
+=pod
+
+=head1 NAME
+
+FS::part_export::cacti
+
+=head1 SYNOPSIS
+
+Cacti integration for Freeside
+
+=head1 DESCRIPTION
+
+This module in particular handles FS::part_export object creation for Cacti integration;
+consult any existing L<FS::part_export> documentation for details on how that works.
+
+=cut
+
 use strict;
 
 use base qw( FS::part_export );
-use FS::Record qw( qsearchs );
+use FS::Record qw( qsearchs qsearch );
 use FS::UID qw( dbh );
+use FS::cacti_page;
 
 use File::Rsync;
-use File::Slurp qw( append_file slurp write_file );
+use File::Slurp qw( slurp );
 use File::stat;
 use MIME::Base64 qw( encode_base64 );
 
@@ -24,8 +42,8 @@ tie my %options, 'Tie::IxHash',
                            default => '' },
   'tree_id'           => { label   => 'Graph Tree ID (optional)',
                            default => '' },
-  'description'       => { label   => 'Description (can use $ip_addr and $description tokens)',
-                           default => 'Freeside $description $ip_addr' },
+  'description'       => { label   => 'Description (can use tokens $contact, $ip_addr and $description)',
+                           default => 'Freeside $contact $description $ip_addr' },
   'graphs_path'       => { label   => 'Graph Export Directory (user@host:/path/to/graphs/)',
                            default => '' },
   'import_freq'       => { label   => 'Minimum minutes between graph imports',
@@ -43,7 +61,7 @@ tie my %options, 'Tie::IxHash',
   'options'         => \%options,
   'notes'           => <<'END',
 Add service to cacti upon provisioning, for broadband services.<BR>
-See FS::part_export::cacti documentation for details.
+See <A HREF="http://www.freeside.biz/mediawiki/index.php/Freeside:4:Documentation:Cacti#Connecting_Cacti_To_Freeside">documentation</A> for details.
 END
 );
 
@@ -113,6 +131,7 @@ sub _insert_queue {
     'tree_id'     => $self->option('tree_id'),
     'description' => $self->option('description'),
 	'svc_desc'    => $svc_broadband->description,
+    'contact'     => $svc_broadband->cust_main->contact,
     'svcnum'      => $svc_broadband->svcnum,
   );
   return ($queue,$error);
@@ -143,12 +162,13 @@ sub ssh_insert {
   die "Non-numerical Host Template ID, check export configuration\n"
     unless $opt{'template_id'} =~ /^\d+$/;
   die "Non-numerical Graph Tree ID, check export configuration\n"
-    unless $opt{'tree_id'} =~ /^\d+$/;
+    unless $opt{'tree_id'} =~ /^\d*$/;
 
   # Add host to cacti
   my $desc = $opt{'description'};
   $desc =~ s/\$ip_addr/$opt{'hostname'}/g;
   $desc =~ s/\$description/$opt{'svc_desc'}/g;
+  $desc =~ s/\$contact/$opt{'contact'}/g;
   $desc =~ s/'/'\\''/g;
   my $cmd = $php
           . $opt{'script_path'} 
@@ -238,11 +258,24 @@ sub ssh_delete {
   return '';
 }
 
-# NOT A METHOD, run as an FS::queue job
-# copies graphs for a single service from Cacti export directory to FS cache
-# generates basic html pages for this service's graphs, and stores them in FS cache
+=head1 SUBROUTINES
+
+=over 4
+
+=item process_graphs JOB PARAM
+
+Intended to be run as an FS::queue job.
+
+Copies graphs for a single service from Cacti export directory to FS cache,
+generates basic html pages for this service with base64-encoded graphs embedded, 
+and stores the generated pages in the database.
+
+=back
+
+=cut
+
 sub process_graphs {
-  my ($job,$param) = @_; #
+  my ($job,$param) = @_;
 
   $job->update_statustext(10);
   my $cachedir = $FS::UID::cache_dir . '/cacti-graphs/';
@@ -259,23 +292,37 @@ sub process_graphs {
 
   $job->update_statustext(20);
 
-  # check for recent uploads, avoid doing this too often
-  my $svchtml = $cachedir.'svc_'.$svcnum.'.html';
-  if (-e $svchtml) {
-    open(my $fh, "<$svchtml");
-    my $firstline = <$fh>;
-    close($fh);
-    if ($firstline =~ /UPDATED (\d+)/) {
-      if ($1 > time - 60 * ($self->option('import_freq') || 5)) {
-        $job->update_statustext(100);
-        return '';
+  my $oldAutoCommit = $FS::UID::AutoCommit;
+  local $FS::UID::AutoCommit = 0;
+  my $dbh = dbh;
+
+  # check for existing pages
+  my $now = time;
+  my @oldpages = qsearch({
+    'table'    => 'cacti_page',
+    'hashref'  => { 'svcnum' => $svcnum, 'exportnum' => $self->exportnum },
+    'select'   => 'cacti_pagenum, exportnum, svcnum, graphnum, imported', #no need to load old content
+    'order_by' => 'ORDER BY graphnum',
+  });
+  if (@oldpages) {
+    #if pages are recent enough, do nothing and return
+    if ($oldpages[0]->imported > $self->exptime($now)) {
+      $job->update_statustext(100);
+      return '';
+    }
+    #delete old pages
+    foreach my $oldpage (@oldpages) {
+      my $error = $oldpage->delete;
+      if ($error) {
+        $dbh->rollback if $oldAutoCommit;
+        die $error;
       }
     }
   }
 
   $job->update_statustext(30);
 
-  # get list of graphs for this svc
+  # get list of graphs for this svc from cacti server
   my $cmd = $php
           . $self->option('script_path')
           . q(freeside_cacti.php --get-graphs --ip=')
@@ -290,7 +337,7 @@ sub process_graphs {
 
   $job->update_statustext(40);
 
-  # copy graphs to cache
+  # copy graphs from cacti server to cache
   # requires version 2.6.4 of rsync, released March 2005
   my $rsync = File::Rsync->new({
     'rsh'       => 'ssh',
@@ -311,12 +358,11 @@ sub process_graphs {
 
   $job->update_statustext(50);
 
-  # create html files in cache
-  my $now = time;
-  my $svchead = q(<!-- UPDATED ) . $now . qq( -->\n)
-              . '<H2 STYLE="margin-top: 0;">Service #' . $svcnum . '</H2>' . "\n"
-              . q(<P>Last updated ) . scalar(localtime($now)) . q(</P>) . "\n";
-  write_file($svchtml,$svchead);
+  # create html file contents
+  my $svchead = q(<!-- UPDATED ) . $now . qq( -->)
+              . '<H2 STYLE="margin-top: 0;">Service #' . $svcnum . '</H2>'
+              . q(<P>Last updated ) . scalar(localtime($now)) . q(</P>);
+  my $svchtml = $svchead;
   my $maxgraph = 1024 * 1024 * ($self->options('max_graph_size') || 5);
   my $nographs = 1;
   for (my $i = 0; $i <= $#graphs; $i++) {
@@ -328,31 +374,53 @@ sub process_graphs {
     ) {
       $nographs = 0;
       # add graph to main file
-      my $graphhead = q(<H3>) . $$graph[1] . q(</H3>) . "\n";
-      append_file( $svchtml, $graphhead,
-        anchor_tag( 
-          $svcnum, $$graph[0], img_tag($thumbfile)
-        )
-      );
+      my $graphhead = q(<H3>) . $$graph[1] . q(</H3>);
+      $svchtml .= $graphhead;
+      $svchtml .= anchor_tag( $svcnum, $$graph[0], img_tag($thumbfile) );
       # create graph details file
-      my $graphhtml = $cachedir . 'svc_' . $svcnum . '_graph_' . $$graph[0] . '.html';
-      write_file($graphhtml,$svchead,$graphhead);
+      my $graphhtml = $svchead . $graphhead;
       my $nodetail = 1;
       my $j = 1;
       while (-e (my $graphfile = $cachedir.'graphs/graph_'.$$graph[0].'_'.$j.'.png')) {
         if ( stat($graphfile)->size() < $maxgraph ) {
           $nodetail = 0;
-          append_file( $graphhtml, img_tag($graphfile) );
+          $graphhtml .= img_tag($graphfile);
         }
         $j++;
       }
-      append_file($graphhtml, '<P>No detail graphs to display for this graph</P>')
+      $graphhtml .= '<P>No detail graphs to display for this graph</P>'
         if $nodetail;
+      my $newobj = new FS::cacti_page {
+        'exportnum' => $self->exportnum,
+        'svcnum'    => $svcnum,
+        'graphnum'  => $$graph[0],
+        'imported'  => $now,
+        'content'   => $graphhtml,
+      };
+      $error = $newobj->insert;
+      if ($error) {
+        $dbh->rollback if $oldAutoCommit;
+        die $error;
+      }
     }
-    $job->update_statustext(50 + ($i / $#graphs) * 50);
+    $job->update_statustext(49 + int($i / $#graphs) * 50);
   }
-  append_file($svchtml,'<P>No graphs to display for this service</P>')
+  $svchtml .= '<P>No graphs to display for this service</P>'
     if $nographs;
+  my $newobj = new FS::cacti_page {
+    'exportnum' => $self->exportnum,
+    'svcnum'    => $svcnum,
+    'graphnum'  => '',
+    'imported'  => $now,
+    'content'   => $svchtml,
+  };
+  $error  = $newobj->insert;
+  if ($error) {
+    $dbh->rollback if $oldAutoCommit;
+    die $error;
+  }
+
+  $dbh->commit or die $dbh->errstr if $oldAutoCommit;
 
   $job->update_statustext(100);
   return '';
@@ -361,8 +429,8 @@ sub process_graphs {
 sub img_tag {
   my $somefile = shift;
   return q(<IMG SRC="data:image/png;base64,)
-       . encode_base64(slurp($somefile,binmode=>':raw'))
-       . qq(" STYLE="margin-bottom: 1em;"><BR>\n);
+       . encode_base64(slurp($somefile,binmode=>':raw'),'')
+       . qq(" STYLE="margin-bottom: 1em;"><BR>);
 }
 
 sub anchor_tag {
@@ -389,82 +457,51 @@ sub ssh_cmd {
   return $output;
 }
 
-=pod
-
-=head1 NAME
-
-FS::part_export::cacti
-
-=head1 SYNOPSIS
-
-Cacti integration for Freeside
-
-=head1 DESCRIPTION
-
-This module in particular handles FS::part_export object creation for Cacti integration;
-consult any existing L<FS::part_export> documentation for details on how that works.
-What follows is more general instructions for connecting your Cacti installation
-to your Freeside installation.
-
-=head2 Connecting Cacti To Freeside
-
-Copy the freeside_cacti.php script from the bin directory of your Freeside
-installation to the cli directory of your Cacti installation.  Give this file 
-the same permissions as the other files in that directory, and create 
-(or choose an existing) user with sufficient permission to read these scripts.
-
-In the regular Cacti interface, create a Host Template to be used by 
-devices exported by Freeside, and note the template's id number.  Optionally,
-create a Graph Tree for these devices to be automatically added to, and note
-the tree's id number.  Configure a Graph Export (under Settings) and note 
-the Export Directory.
-
-In Freeside, go to Configuration->Services->Provisioning exports to
-add a new export.  From the Add Export page, select cacti for Export then enter...
-
-* the Hostname or IP address of your Cacti server
-
-* the User Name with permission to run scripts in the cli directory
-
-* the full Script Path to that directory (eg /usr/share/cacti/cli/)
+=head1 METHODS
 
-* the Host Template ID for adding new devices
+=over 4
 
-* the Graph Tree ID for adding new devices (optional)
+=item cleanup
 
-* the Description for new devices;  you can use the tokens
-  $ip_addr and $description to include the equivalent fields
-  from the broadband service definition
+Removes all expired graphs for this export from the database.
 
-* the Graph Export Directory, including connection information
-  if necessary (user@host:/path/to/graphs/)
+=cut
 
-* the minimum minutes between graph imports to Freeside (graphs will
-  otherwise be imported into Freeside as needed.)  This should be at least
-  as long as the minumum time between graph exports configured in Cacti.
-  Defaults to 5 if unspecified.
+sub cleanup {
+  my $self = shift;
+  my $oldAutoCommit = $FS::UID::AutoCommit;
+  local $FS::UID::AutoCommit = 0;
+  my $dbh = dbh;
+  my $sth = $dbh->prepare('DELETE FROM cacti_page WHERE exportnum = ? and imported <= ?') 
+    or do {
+      $dbh->rollback if $oldAutoCommit;
+      return $dbh->errstr;
+    };
+  $sth->execute($self->exportnum,$self->exptime)
+    or do {
+      $dbh->rollback if $oldAutoCommit;
+      return $dbh->errstr;
+    };
+  $dbh->commit or return $dbh->errstr if $oldAutoCommit;
+  return '';
+}
 
-* the maximum size per graph, in MB;  individual graphs that exceed this size
-  will be quietly ignored by Freeside.  Defaults to 5 if unspecified.
+=item exptime [ TIME ]
 
-After adding the export, go to Configuration->Services->Service definitions.
-The export you just created will be available for selection when adding or
-editing broadband service definitions; check the box to activate it for 
-a given service.  Note that you should only have one cacti export per
-broadband service definition.
+Accepts optional current time, defaults to actual current time.
 
-When properly configured broadband services are provisioned, they will now
-be added to Cacti using the Host Template you specified.  If you also specified
-a Graph Tree, the created device will also be added to that.
+Returns timestamp for the oldest possible non-expired graph import,
+based on the import_freq option.
 
-Once added, a link to the graphs for this host will be available when viewing 
-the details of the provisioned service in Freeside.
+=cut
 
-Devices will be deleted from Cacti when the service is unprovisioned in Freeside, 
-and they will be deleted and re-added if the ip address changes.
+sub exptime {
+  my $self = shift;
+  my $now = shift || time;
+  return $now - 60 * ($self->option('import_freq') || 5);
+}
 
-Currently, graphs themselves must still be added in Cacti by hand or some
-other form of automation tailored to your specific graph inputs and data sources.
+=back
 
 =head1 AUTHOR