RT#18834: Cacti integration [real graph import]

6 files changed, 263 insertions, 48 deletions

diff --git a/FS/FS/Schema.pm b/FS/FS/Schema.pm
index 3cdad433a..a048d3e24 100644
--- a/FS/FS/Schema.pm
+++ b/FS/FS/Schema.pm
@@ -4677,7 +4677,6 @@ sub tables_hashref {
         'suid',                    'int', 'NULL',        '', '', '',
         'shared_svcnum',           'int', 'NULL',        '', '', '',
         'serviceid',           'varchar', 'NULL',        64, '', '',#srvexport/reportfields
-        'cacti_leaf_id',           'int', 'NULL',        '', '', '',
       ],
       'primary_key'  => 'svcnum',
       'unique'       => [ [ 'ip_addr' ], [ 'mac_addr' ] ],
diff --git a/FS/FS/part_export/cacti.pm b/FS/FS/part_export/cacti.pm
index 6877c8f5f..1f5f64c2a 100644
--- a/FS/FS/part_export/cacti.pm
+++ b/FS/FS/part_export/cacti.pm
@@ -1,10 +1,16 @@
 package FS::part_export::cacti;
 
 use strict;
+
 use base qw( FS::part_export );
 use FS::Record qw( qsearchs );
 use FS::UID qw( dbh );
 
+use File::Rsync;
+use File::Slurp qw( append_file slurp write_file );
+use File::stat;
+use MIME::Base64 qw( encode_base64 );
+
 use vars qw( %info );
 
 my $php = 'php -q ';
@@ -14,14 +20,18 @@ tie my %options, 'Tie::IxHash',
                            default => 'freeside' },
   'script_path'       => { label   => 'Script Path',
                            default => '/usr/share/cacti/cli/' },
-  'base_url'          => { label   => 'Base Cacti URL',
-                           default => '' },
   'template_id'       => { label   => 'Host Template ID',
                            default => '' },
-  'tree_id'           => { label   => 'Graph Tree ID',
+  'tree_id'           => { label   => 'Graph Tree ID (optional)',
                            default => '' },
   'description'       => { label   => 'Description (can use $ip_addr and $description tokens)',
                            default => 'Freeside $description $ip_addr' },
+  'graphs_path'       => { label   => 'Graph Export Directory (user@host:/path/to/graphs/)',
+                           default => '' },
+  'import_freq'       => { label   => 'Minimum minutes between graph imports',
+                           default => '5' },
+  'max_graph_size'    => { label   => 'Maximum size per graph (MB)',
+                           default => '5' },
 #  'delete_graphs'     => { label   => 'Delete associated graphs and data sources when unprovisioning', 
 #                           type    => 'checkbox',
 #                         },
@@ -155,27 +165,18 @@ sub ssh_insert {
   my $id = $1;
 
   # Add host to tree
-  $cmd = $php
-       . $opt{'script_path'}
-       . q(add_tree.php --type=node --node-type=host --tree-id=)
-       . $opt{'tree_id'}
-       . q( --host-id=)
-       . $id;
-  $response = ssh_cmd(%opt, 'command' => $cmd);
-  unless ( $response =~ /Added Node node-id: \((\d+)\)/ ) {
+  if ($opt{'tree_id'}) {
+    $cmd = $php
+         . $opt{'script_path'}
+         . q(add_tree.php --type=node --node-type=host --tree-id=)
+         . $opt{'tree_id'}
+         . q( --host-id=)
+         . $id;
+    $response = ssh_cmd(%opt, 'command' => $cmd);
+    unless ( $response =~ /Added Node node-id: \((\d+)\)/ ) {
       die "Error adding host to tree: $response";
+    }
   }
-  my $leaf_id = $1;
-
-  # Store id for generating graph urls
-  my $svc_broadband = qsearchs({
-    'table'   => 'svc_broadband',
-    'hashref' => { 'svcnum' => $opt{'svcnum'} },
-  });
-  die "Could not reload broadband service" unless $svc_broadband;
-  $svc_broadband->set('cacti_leaf_id',$leaf_id);
-  my $error = $svc_broadband->replace;
-  return $error if $error;
 
 #  # Get list of graph templates for new id
 #  $cmd = $php
@@ -237,6 +238,145 @@ sub ssh_delete {
   return '';
 }
 
+# NOT A METHOD, run as an FS::queue job
+# copies graphs for a single service from Cacti export directory to FS cache
+# generates basic html pages for this service's graphs, and stores them in FS cache
+sub process_graphs {
+  my ($job,$param) = @_; #
+
+  $job->update_statustext(10);
+  my $cachedir = $FS::UID::cache_dir . '/cacti-graphs/';
+
+  # load the service
+  my $svcnum = $param->{'svcnum'} || die "No svcnum specified";
+  my $svc = qsearchs({
+   'table'   => 'svc_broadband',
+   'hashref' => { 'svcnum' => $svcnum },
+  }) || die "Could not load svcnum $svcnum";
+
+  # load relevant FS::part_export::cacti object
+  my ($self) = $svc->cust_svc->part_svc->part_export('cacti');
+
+  $job->update_statustext(20);
+
+  # check for recent uploads, avoid doing this too often
+  my $svchtml = $cachedir.'svc_'.$svcnum.'.html';
+  if (-e $svchtml) {
+    open(my $fh, "<$svchtml");
+    my $firstline = <$fh>;
+    close($fh);
+    if ($firstline =~ /UPDATED (\d+)/) {
+      if ($1 > time - 60 * ($self->option('import_freq') || 5)) {
+        $job->update_statustext(100);
+        return '';
+      }
+    }
+  }
+
+  $job->update_statustext(30);
+
+  # get list of graphs for this svc
+  my $cmd = $php
+          . $self->option('script_path')
+          . q(freeside_cacti.php --get-graphs --ip=')
+          . $svc->ip_addr
+          . q(');
+  my @graphs = map { [ split(/\t/,$_) ] } 
+                 split(/\n/, ssh_cmd(
+                   'host'          => $self->machine,
+                   'user'          => $self->option('user'),
+                   'command'       => $cmd
+                 ));
+
+  $job->update_statustext(40);
+
+  # copy graphs to cache
+  # requires version 2.6.4 of rsync, released March 2005
+  my $rsync = File::Rsync->new({
+    'rsh'       => 'ssh',
+    'verbose'   => 1,
+    'recursive' => 1,
+    'source'    => $self->option('graphs_path'),
+    'dest'      => $cachedir,
+    'include'   => [
+      (map { q('**graph_).${$_}[0].q(*.png') } @graphs),
+      (map { q('**thumb_).${$_}[0].q(.png') } @graphs),
+      q('*/'),
+      q('- *'),
+    ],
+  });
+  #don't know why a regular $rsync->exec isn't doing includes right, but this does
+  my $error = system(join(' ',@{$rsync->getcmd()}));
+  die "rsync failed with exit status $error" if $error;
+
+  $job->update_statustext(50);
+
+  # create html files in cache
+  my $now = time;
+  my $svchead = q(<!-- UPDATED ) . $now . qq( -->\n)
+              . '<H2 STYLE="margin-top: 0;">Service #' . $svcnum . '</H2>' . "\n"
+              . q(<P>Last updated ) . scalar(localtime($now)) . q(</P>) . "\n";
+  write_file($svchtml,$svchead);
+  my $maxgraph = 1024 * 1024 * ($self->options('max_graph_size') || 5);
+  my $nographs = 1;
+  for (my $i = 0; $i <= $#graphs; $i++) {
+    my $graph = $graphs[$i];
+    my $thumbfile = $cachedir . 'graphs/thumb_' . $$graph[0] . '.png';
+    if (
+      (-e $thumbfile) && 
+      ( stat($thumbfile)->size() < $maxgraph )
+    ) {
+      $nographs = 0;
+      # add graph to main file
+      my $graphhead = q(<H3>) . $$graph[1] . q(</H3>) . "\n";
+      append_file( $svchtml, $graphhead,
+        anchor_tag( 
+          $svcnum, $$graph[0], img_tag($thumbfile)
+        )
+      );
+      # create graph details file
+      my $graphhtml = $cachedir . 'svc_' . $svcnum . '_graph_' . $$graph[0] . '.html';
+      write_file($graphhtml,$svchead,$graphhead);
+      my $nodetail = 1;
+      my $j = 1;
+      while (-e (my $graphfile = $cachedir.'graphs/graph_'.$$graph[0].'_'.$j.'.png')) {
+        if ( stat($graphfile)->size() < $maxgraph ) {
+          $nodetail = 0;
+          append_file( $graphhtml, img_tag($graphfile) );
+        }
+        $j++;
+      }
+      append_file($graphhtml, '<P>No detail graphs to display for this graph</P>')
+        if $nodetail;
+    }
+    $job->update_statustext(50 + ($i / $#graphs) * 50);
+  }
+  append_file($svchtml,'<P>No graphs to display for this service</P>')
+    if $nographs;
+
+  $job->update_statustext(100);
+  return '';
+}
+
+sub img_tag {
+  my $somefile = shift;
+  return q(<IMG SRC="data:image/png;base64,)
+       . encode_base64(slurp($somefile,binmode=>':raw'))
+       . qq(" STYLE="margin-bottom: 1em;"><BR>\n);
+}
+
+sub anchor_tag {
+  my ($svcnum, $graphnum, $contents) = @_;
+  return q(<A HREF="?svcnum=)
+       . $svcnum
+       . q(&graphnum=)
+       . $graphnum
+       . q(">)
+       . $contents
+       . q(</A>);
+}
+
+#this gets used by everything else
 #fake false laziness, other ssh_cmds handle error/output differently
 sub ssh_cmd {
   use Net::OpenSSH;
@@ -274,41 +414,56 @@ the same permissions as the other files in that directory, and create
 (or choose an existing) user with sufficient permission to read these scripts.
 
 In the regular Cacti interface, create a Host Template to be used by 
-devices exported by Freeside, and note the template's id number.
+devices exported by Freeside, and note the template's id number.  Optionally,
+create a Graph Tree for these devices to be automatically added to, and note
+the tree's id number.  Configure a Graph Export (under Settings) and note 
+the Export Directory.
 
 In Freeside, go to Configuration->Services->Provisioning exports to
 add a new export.  From the Add Export page, select cacti for Export then enter...
 
-* the User Name with permission to run scripts in the cli directory
+* the Hostname or IP address of your Cacti server
 
-* enter the full Script Path to that directory (eg /usr/share/cacti/cli/)
+* the User Name with permission to run scripts in the cli directory
 
-* enter the Base Cacti URL for your cacti server (eg https://example.com/cacti/)
+* the full Script Path to that directory (eg /usr/share/cacti/cli/)
 
 * the Host Template ID for adding new devices
 
-* the Graph Tree ID for adding new devices
+* the Graph Tree ID for adding new devices (optional)
 
 * the Description for new devices;  you can use the tokens
   $ip_addr and $description to include the equivalent fields
   from the broadband service definition
 
+* the Graph Export Directory, including connection information
+  if necessary (user@host:/path/to/graphs/)
+
+* the minimum minutes between graph imports to Freeside (graphs will
+  otherwise be imported into Freeside as needed.)  This should be at least
+  as long as the minumum time between graph exports configured in Cacti.
+  Defaults to 5 if unspecified.
+
+* the maximum size per graph, in MB;  individual graphs that exceed this size
+  will be quietly ignored by Freeside.  Defaults to 5 if unspecified.
+
 After adding the export, go to Configuration->Services->Service definitions.
 The export you just created will be available for selection when adding or
-editing broadband service definitions.
+editing broadband service definitions; check the box to activate it for 
+a given service.  Note that you should only have one cacti export per
+broadband service definition.
 
-When properly configured broadband services are provisioned, they should now
-be added to Cacti using the Host Template you specified, and the created device
-will also be added to the specified Graph Tree.
+When properly configured broadband services are provisioned, they will now
+be added to Cacti using the Host Template you specified.  If you also specified
+a Graph Tree, the created device will also be added to that.
 
 Once added, a link to the graphs for this host will be available when viewing 
-the details of the provisioned service in Freeside (you will need to authenticate 
-into Cacti to view them.)
+the details of the provisioned service in Freeside.
 
 Devices will be deleted from Cacti when the service is unprovisioned in Freeside, 
 and they will be deleted and re-added if the ip address changes.
 
-Currently, graphs themselves must still be added in cacti by hand or some
+Currently, graphs themselves must still be added in Cacti by hand or some
 other form of automation tailored to your specific graph inputs and data sources.
 
 =head1 AUTHOR
@@ -320,8 +475,8 @@ jonathan@freeside.biz
 
 Copyright 2015 Freeside Internet Services      
 
-This program is free software; you can redistribute it and/or           |
-modify it under the terms of the GNU General Public License             |
+This program is free software; you can redistribute it and/or 
+modify it under the terms of the GNU General Public License 
 as published by the Free Software Foundation.
 
 =cut
diff --git a/bin/freeside_cacti.php b/bin/freeside_cacti.php
index 22fb0f0f4..0a9ee9c1c 100755
--- a/bin/freeside_cacti.php
+++ b/bin/freeside_cacti.php
@@ -32,15 +32,15 @@ if (!isset($_SERVER["argv"][0]) || isset($_SERVER['REQUEST_METHOD'])  || isset($
 $no_http_headers = true;
 
 /* 
-Currently, only drop-device is actually being used by Freeside integration,
+Currently, only drop-device and get-graphs is actually being used by Freeside integration,
 but keeping commented out code for potential future development.
 */
 
 include(dirname(__FILE__)."/../site/include/global.php");
 include_once($config["base_path"]."/lib/api_device.php");
+include_once($config["base_path"]."/lib/api_automation_tools.php");
 
 /*
-include_once($config["base_path"]."/lib/api_automation_tools.php");
 include_once($config["base_path"]."/lib/api_data_source.php");
 include_once($config["base_path"]."/lib/api_graph.php");
 include_once($config["base_path"]."/lib/functions.php");
@@ -57,6 +57,9 @@ if (sizeof($parms)) {
 	foreach($parms as $parameter) {
 		@list($arg, $value) = @explode("=", $parameter);
 		switch ($arg) {
+        case "--get-graphs":
+			$action = 'get-graphs';
+            break;
         case "--drop-device":
 			$action = 'drop-device';
             break;
@@ -94,6 +97,9 @@ if (sizeof($parms)) {
 
 /* Now take an action */
 switch ($action) {
+case "get-graphs":
+	displayHostGraphs(host_id($ip),TRUE);
+	break;
 case "drop-device":
 	$host_id = host_id($ip);
 /*
diff --git a/httemplate/misc/cacti_graphs.html b/httemplate/misc/cacti_graphs.html
new file mode 100644
index 000000000..9cc5e2494
--- /dev/null
+++ b/httemplate/misc/cacti_graphs.html
@@ -0,0 +1,53 @@
+<% include( '/elements/header.html', 'Cacti Graphs' ) %>
+
+% if ($load) {
+
+<FORM NAME="CactiGraphForm" ID="CactiGraphForm" style="margin-top: 0">
+<INPUT TYPE="hidden" NAME="svcnum" VALUE="<% $svcnum %>">
+</FORM>
+<% include( '/elements/progress-init.html',
+              'CactiGraphForm', 
+              [ 'svcnum' ],
+              $p.'misc/process/cacti_graphs.cgi',
+              { url => 'javascript:window.location.replace("'.popurl(2).'misc/cacti_graphs.html?svcnum='.$svcnum.'")' },
+) %>
+<!--
+  note we use window.location.replace for the callback url above
+  so that this page gets removed from browser history after processing
+  so that process() doesn't get triggered by the back button
+-->
+<P>Loading graphs, please wait...</P>
+<SCRIPT TYPE="text/javascript">
+process();
+</SCRIPT>
+
+% } else {
+%   if ($error) {
+
+<P><% $error %></P>
+
+%   } else {
+
+<% slurp($htmlfile) %>
+
+%   }
+% }
+
+<%init>
+use File::Slurp qw( slurp );
+
+my $svcnum    = $cgi->param('svcnum') or die 'Illegal svcnum';
+my $load      = $cgi->param('load');
+my $graphnum  = $cgi->param('graphnum');
+
+my $htmlfile = $FS::UID::cache_dir 
+             . '/cacti-graphs/'
+             . 'svc_'
+             . $svcnum;
+$htmlfile .= '_graph_' . $graphnum
+  if $graphnum;
+$htmlfile .= '.html';
+
+my $error = (-e $htmlfile) ? '' : 'File not found';
+</%init>
+
diff --git a/httemplate/misc/process/cacti_graphs.cgi b/httemplate/misc/process/cacti_graphs.cgi
new file mode 100644
index 000000000..160b1ad85
--- /dev/null
+++ b/httemplate/misc/process/cacti_graphs.cgi
@@ -0,0 +1,6 @@
+<% $server->process %>
+
+<%init>
+my $server = FS::UI::Web::JSRPC->new('FS::part_export::cacti::process_graphs', $cgi);
+</%init>
+
diff --git a/httemplate/view/svc_broadband.cgi b/httemplate/view/svc_broadband.cgi
index 9fe10bd3a..4935a1096 100644
--- a/httemplate/view/svc_broadband.cgi
+++ b/httemplate/view/svc_broadband.cgi
@@ -72,15 +72,11 @@ sub ip_addr {
   my $out = $ip_addr;
   $out .= ' (' . include('/elements/popup_link-ping.html', ip => $ip_addr) . ')'
     if $ip_addr;
-  if ($svc->cacti_leaf_id) {
-    # should only ever be one, but not sure if that is enforced
-    my ($cacti) = $svc->cust_svc->part_svc->part_export('cacti');
-    $out .= ' (<A HREF="' 
-         .  $cacti->option('base_url')
-         .  'graph_view.php?action=tree&tree_id='
-         .  $cacti->option('tree_id')
-         .  '&leaf_id='
-         .  $svc->cacti_leaf_id
+  if ($svc->cust_svc->part_svc->part_export('cacti')) {
+    $out .= ' (<A HREF="'
+         .  popurl(2)
+         .  'misc/cacti_graphs.html?load=1&svcnum=' 
+         .  $svc->svcnum
          .  '">cacti</A>)';
   }
   if ( my $addr_block = $svc->addr_block ) {