From: Jonathan Prykop Date: Fri, 27 Mar 2015 19:20:48 +0000 (-0500) Subject: RT#18834: Cacti integration [real graph import] X-Git-Url: http://git.freeside.biz/gitweb/?p=freeside.git;a=commitdiff_plain;h=d9db63d82fce670cc3c21f86e577dd99c3d14028 RT#18834: Cacti integration [real graph import] --- diff --git a/FS/FS/Schema.pm b/FS/FS/Schema.pm index 3cdad433a..a048d3e24 100644 --- a/FS/FS/Schema.pm +++ b/FS/FS/Schema.pm @@ -4677,7 +4677,6 @@ sub tables_hashref { 'suid', 'int', 'NULL', '', '', '', 'shared_svcnum', 'int', 'NULL', '', '', '', 'serviceid', 'varchar', 'NULL', 64, '', '',#srvexport/reportfields - 'cacti_leaf_id', 'int', 'NULL', '', '', '', ], 'primary_key' => 'svcnum', 'unique' => [ [ 'ip_addr' ], [ 'mac_addr' ] ], diff --git a/FS/FS/part_export/cacti.pm b/FS/FS/part_export/cacti.pm index 6877c8f5f..1f5f64c2a 100644 --- a/FS/FS/part_export/cacti.pm +++ b/FS/FS/part_export/cacti.pm @@ -1,10 +1,16 @@ package FS::part_export::cacti; use strict; + use base qw( FS::part_export ); use FS::Record qw( qsearchs ); use FS::UID qw( dbh ); +use File::Rsync; +use File::Slurp qw( append_file slurp write_file ); +use File::stat; +use MIME::Base64 qw( encode_base64 ); + use vars qw( %info ); my $php = 'php -q '; @@ -14,14 +20,18 @@ tie my %options, 'Tie::IxHash', default => 'freeside' }, 'script_path' => { label => 'Script Path', default => '/usr/share/cacti/cli/' }, - 'base_url' => { label => 'Base Cacti URL', - default => '' }, 'template_id' => { label => 'Host Template ID', default => '' }, - 'tree_id' => { label => 'Graph Tree ID', + 'tree_id' => { label => 'Graph Tree ID (optional)', default => '' }, 'description' => { label => 'Description (can use $ip_addr and $description tokens)', default => 'Freeside $description $ip_addr' }, + 'graphs_path' => { label => 'Graph Export Directory (user@host:/path/to/graphs/)', + default => '' }, + 'import_freq' => { label => 'Minimum minutes between graph imports', + default => '5' }, + 'max_graph_size' => { label => 'Maximum size per graph (MB)', + default => '5' }, # 'delete_graphs' => { label => 'Delete associated graphs and data sources when unprovisioning', # type => 'checkbox', # }, @@ -155,27 +165,18 @@ sub ssh_insert { my $id = $1; # Add host to tree - $cmd = $php - . $opt{'script_path'} - . q(add_tree.php --type=node --node-type=host --tree-id=) - . $opt{'tree_id'} - . q( --host-id=) - . $id; - $response = ssh_cmd(%opt, 'command' => $cmd); - unless ( $response =~ /Added Node node-id: \((\d+)\)/ ) { + if ($opt{'tree_id'}) { + $cmd = $php + . $opt{'script_path'} + . q(add_tree.php --type=node --node-type=host --tree-id=) + . $opt{'tree_id'} + . q( --host-id=) + . $id; + $response = ssh_cmd(%opt, 'command' => $cmd); + unless ( $response =~ /Added Node node-id: \((\d+)\)/ ) { die "Error adding host to tree: $response"; + } } - my $leaf_id = $1; - - # Store id for generating graph urls - my $svc_broadband = qsearchs({ - 'table' => 'svc_broadband', - 'hashref' => { 'svcnum' => $opt{'svcnum'} }, - }); - die "Could not reload broadband service" unless $svc_broadband; - $svc_broadband->set('cacti_leaf_id',$leaf_id); - my $error = $svc_broadband->replace; - return $error if $error; # # Get list of graph templates for new id # $cmd = $php @@ -237,6 +238,145 @@ sub ssh_delete { return ''; } +# NOT A METHOD, run as an FS::queue job +# copies graphs for a single service from Cacti export directory to FS cache +# generates basic html pages for this service's graphs, and stores them in FS cache +sub process_graphs { + my ($job,$param) = @_; # + + $job->update_statustext(10); + my $cachedir = $FS::UID::cache_dir . '/cacti-graphs/'; + + # load the service + my $svcnum = $param->{'svcnum'} || die "No svcnum specified"; + my $svc = qsearchs({ + 'table' => 'svc_broadband', + 'hashref' => { 'svcnum' => $svcnum }, + }) || die "Could not load svcnum $svcnum"; + + # load relevant FS::part_export::cacti object + my ($self) = $svc->cust_svc->part_svc->part_export('cacti'); + + $job->update_statustext(20); + + # check for recent uploads, avoid doing this too often + my $svchtml = $cachedir.'svc_'.$svcnum.'.html'; + if (-e $svchtml) { + open(my $fh, "<$svchtml"); + my $firstline = <$fh>; + close($fh); + if ($firstline =~ /UPDATED (\d+)/) { + if ($1 > time - 60 * ($self->option('import_freq') || 5)) { + $job->update_statustext(100); + return ''; + } + } + } + + $job->update_statustext(30); + + # get list of graphs for this svc + my $cmd = $php + . $self->option('script_path') + . q(freeside_cacti.php --get-graphs --ip=') + . $svc->ip_addr + . q('); + my @graphs = map { [ split(/\t/,$_) ] } + split(/\n/, ssh_cmd( + 'host' => $self->machine, + 'user' => $self->option('user'), + 'command' => $cmd + )); + + $job->update_statustext(40); + + # copy graphs to cache + # requires version 2.6.4 of rsync, released March 2005 + my $rsync = File::Rsync->new({ + 'rsh' => 'ssh', + 'verbose' => 1, + 'recursive' => 1, + 'source' => $self->option('graphs_path'), + 'dest' => $cachedir, + 'include' => [ + (map { q('**graph_).${$_}[0].q(*.png') } @graphs), + (map { q('**thumb_).${$_}[0].q(.png') } @graphs), + q('*/'), + q('- *'), + ], + }); + #don't know why a regular $rsync->exec isn't doing includes right, but this does + my $error = system(join(' ',@{$rsync->getcmd()})); + die "rsync failed with exit status $error" if $error; + + $job->update_statustext(50); + + # create html files in cache + my $now = time; + my $svchead = q(\n) + . '

Service #' . $svcnum . '

' . "\n" + . q(

Last updated ) . scalar(localtime($now)) . q(

) . "\n"; + write_file($svchtml,$svchead); + my $maxgraph = 1024 * 1024 * ($self->options('max_graph_size') || 5); + my $nographs = 1; + for (my $i = 0; $i <= $#graphs; $i++) { + my $graph = $graphs[$i]; + my $thumbfile = $cachedir . 'graphs/thumb_' . $$graph[0] . '.png'; + if ( + (-e $thumbfile) && + ( stat($thumbfile)->size() < $maxgraph ) + ) { + $nographs = 0; + # add graph to main file + my $graphhead = q(

) . $$graph[1] . q(

) . "\n"; + append_file( $svchtml, $graphhead, + anchor_tag( + $svcnum, $$graph[0], img_tag($thumbfile) + ) + ); + # create graph details file + my $graphhtml = $cachedir . 'svc_' . $svcnum . '_graph_' . $$graph[0] . '.html'; + write_file($graphhtml,$svchead,$graphhead); + my $nodetail = 1; + my $j = 1; + while (-e (my $graphfile = $cachedir.'graphs/graph_'.$$graph[0].'_'.$j.'.png')) { + if ( stat($graphfile)->size() < $maxgraph ) { + $nodetail = 0; + append_file( $graphhtml, img_tag($graphfile) ); + } + $j++; + } + append_file($graphhtml, '

No detail graphs to display for this graph

') + if $nodetail; + } + $job->update_statustext(50 + ($i / $#graphs) * 50); + } + append_file($svchtml,'

No graphs to display for this service

') + if $nographs; + + $job->update_statustext(100); + return ''; +} + +sub img_tag { + my $somefile = shift; + return q(
\n); +} + +sub anchor_tag { + my ($svcnum, $graphnum, $contents) = @_; + return q() + . $contents + . q(); +} + +#this gets used by everything else #fake false laziness, other ssh_cmds handle error/output differently sub ssh_cmd { use Net::OpenSSH; @@ -274,41 +414,56 @@ the same permissions as the other files in that directory, and create (or choose an existing) user with sufficient permission to read these scripts. In the regular Cacti interface, create a Host Template to be used by -devices exported by Freeside, and note the template's id number. +devices exported by Freeside, and note the template's id number. Optionally, +create a Graph Tree for these devices to be automatically added to, and note +the tree's id number. Configure a Graph Export (under Settings) and note +the Export Directory. In Freeside, go to Configuration->Services->Provisioning exports to add a new export. From the Add Export page, select cacti for Export then enter... -* the User Name with permission to run scripts in the cli directory +* the Hostname or IP address of your Cacti server -* enter the full Script Path to that directory (eg /usr/share/cacti/cli/) +* the User Name with permission to run scripts in the cli directory -* enter the Base Cacti URL for your cacti server (eg https://example.com/cacti/) +* the full Script Path to that directory (eg /usr/share/cacti/cli/) * the Host Template ID for adding new devices -* the Graph Tree ID for adding new devices +* the Graph Tree ID for adding new devices (optional) * the Description for new devices; you can use the tokens $ip_addr and $description to include the equivalent fields from the broadband service definition +* the Graph Export Directory, including connection information + if necessary (user@host:/path/to/graphs/) + +* the minimum minutes between graph imports to Freeside (graphs will + otherwise be imported into Freeside as needed.) This should be at least + as long as the minumum time between graph exports configured in Cacti. + Defaults to 5 if unspecified. + +* the maximum size per graph, in MB; individual graphs that exceed this size + will be quietly ignored by Freeside. Defaults to 5 if unspecified. + After adding the export, go to Configuration->Services->Service definitions. The export you just created will be available for selection when adding or -editing broadband service definitions. +editing broadband service definitions; check the box to activate it for +a given service. Note that you should only have one cacti export per +broadband service definition. -When properly configured broadband services are provisioned, they should now -be added to Cacti using the Host Template you specified, and the created device -will also be added to the specified Graph Tree. +When properly configured broadband services are provisioned, they will now +be added to Cacti using the Host Template you specified. If you also specified +a Graph Tree, the created device will also be added to that. Once added, a link to the graphs for this host will be available when viewing -the details of the provisioned service in Freeside (you will need to authenticate -into Cacti to view them.) +the details of the provisioned service in Freeside. Devices will be deleted from Cacti when the service is unprovisioned in Freeside, and they will be deleted and re-added if the ip address changes. -Currently, graphs themselves must still be added in cacti by hand or some +Currently, graphs themselves must still be added in Cacti by hand or some other form of automation tailored to your specific graph inputs and data sources. =head1 AUTHOR @@ -320,8 +475,8 @@ jonathan@freeside.biz Copyright 2015 Freeside Internet Services -This program is free software; you can redistribute it and/or | -modify it under the terms of the GNU General Public License | +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License as published by the Free Software Foundation. =cut diff --git a/bin/freeside_cacti.php b/bin/freeside_cacti.php index 22fb0f0f4..0a9ee9c1c 100755 --- a/bin/freeside_cacti.php +++ b/bin/freeside_cacti.php @@ -32,15 +32,15 @@ if (!isset($_SERVER["argv"][0]) || isset($_SERVER['REQUEST_METHOD']) || isset($ $no_http_headers = true; /* -Currently, only drop-device is actually being used by Freeside integration, +Currently, only drop-device and get-graphs is actually being used by Freeside integration, but keeping commented out code for potential future development. */ include(dirname(__FILE__)."/../site/include/global.php"); include_once($config["base_path"]."/lib/api_device.php"); +include_once($config["base_path"]."/lib/api_automation_tools.php"); /* -include_once($config["base_path"]."/lib/api_automation_tools.php"); include_once($config["base_path"]."/lib/api_data_source.php"); include_once($config["base_path"]."/lib/api_graph.php"); include_once($config["base_path"]."/lib/functions.php"); @@ -57,6 +57,9 @@ if (sizeof($parms)) { foreach($parms as $parameter) { @list($arg, $value) = @explode("=", $parameter); switch ($arg) { + case "--get-graphs": + $action = 'get-graphs'; + break; case "--drop-device": $action = 'drop-device'; break; @@ -94,6 +97,9 @@ if (sizeof($parms)) { /* Now take an action */ switch ($action) { +case "get-graphs": + displayHostGraphs(host_id($ip),TRUE); + break; case "drop-device": $host_id = host_id($ip); /* diff --git a/httemplate/misc/cacti_graphs.html b/httemplate/misc/cacti_graphs.html new file mode 100644 index 000000000..9cc5e2494 --- /dev/null +++ b/httemplate/misc/cacti_graphs.html @@ -0,0 +1,53 @@ +<% include( '/elements/header.html', 'Cacti Graphs' ) %> + +% if ($load) { + +
+ +
+<% include( '/elements/progress-init.html', + 'CactiGraphForm', + [ 'svcnum' ], + $p.'misc/process/cacti_graphs.cgi', + { url => 'javascript:window.location.replace("'.popurl(2).'misc/cacti_graphs.html?svcnum='.$svcnum.'")' }, +) %> + +

Loading graphs, please wait...

+ + +% } else { +% if ($error) { + +

<% $error %>

+ +% } else { + +<% slurp($htmlfile) %> + +% } +% } + +<%init> +use File::Slurp qw( slurp ); + +my $svcnum = $cgi->param('svcnum') or die 'Illegal svcnum'; +my $load = $cgi->param('load'); +my $graphnum = $cgi->param('graphnum'); + +my $htmlfile = $FS::UID::cache_dir + . '/cacti-graphs/' + . 'svc_' + . $svcnum; +$htmlfile .= '_graph_' . $graphnum + if $graphnum; +$htmlfile .= '.html'; + +my $error = (-e $htmlfile) ? '' : 'File not found'; + + diff --git a/httemplate/misc/process/cacti_graphs.cgi b/httemplate/misc/process/cacti_graphs.cgi new file mode 100644 index 000000000..160b1ad85 --- /dev/null +++ b/httemplate/misc/process/cacti_graphs.cgi @@ -0,0 +1,6 @@ +<% $server->process %> + +<%init> +my $server = FS::UI::Web::JSRPC->new('FS::part_export::cacti::process_graphs', $cgi); + + diff --git a/httemplate/view/svc_broadband.cgi b/httemplate/view/svc_broadband.cgi index 9fe10bd3a..4935a1096 100644 --- a/httemplate/view/svc_broadband.cgi +++ b/httemplate/view/svc_broadband.cgi @@ -72,15 +72,11 @@ sub ip_addr { my $out = $ip_addr; $out .= ' (' . include('/elements/popup_link-ping.html', ip => $ip_addr) . ')' if $ip_addr; - if ($svc->cacti_leaf_id) { - # should only ever be one, but not sure if that is enforced - my ($cacti) = $svc->cust_svc->part_svc->part_export('cacti'); - $out .= ' (svcnum . '">cacti)'; } if ( my $addr_block = $svc->addr_block ) {