X-Git-Url: http://git.freeside.biz/gitweb/?a=blobdiff_plain;f=FS%2FFS%2Fpart_export%2Fcacti.pm;h=31c0dc53707000c32ddd3cc6d085d912eb46397f;hb=ffa18709ee8a4d05e18d2d406cf73afe79e52524;hp=6877c8f5f9ad0e20f69eac163898fc29d1b52938;hpb=5f4099e52bd894d644c676ea75e1b0cb588393c8;p=freeside.git diff --git a/FS/FS/part_export/cacti.pm b/FS/FS/part_export/cacti.pm index 6877c8f5f..31c0dc537 100644 --- a/FS/FS/part_export/cacti.pm +++ b/FS/FS/part_export/cacti.pm @@ -1,9 +1,33 @@ package FS::part_export::cacti; +=pod + +=head1 NAME + +FS::part_export::cacti + +=head1 SYNOPSIS + +Cacti integration for Freeside + +=head1 DESCRIPTION + +This module in particular handles FS::part_export object creation for Cacti integration; +consult any existing L documentation for details on how that works. + +=cut + use strict; + use base qw( FS::part_export ); -use FS::Record qw( qsearchs ); +use FS::Record qw( qsearchs qsearch ); use FS::UID qw( dbh ); +use FS::cacti_page; + +use File::Rsync; +use File::Slurp qw( slurp ); +use File::stat; +use MIME::Base64 qw( encode_base64 ); use vars qw( %info ); @@ -14,26 +38,58 @@ tie my %options, 'Tie::IxHash', default => 'freeside' }, 'script_path' => { label => 'Script Path', default => '/usr/share/cacti/cli/' }, - 'base_url' => { label => 'Base Cacti URL', - default => '' }, 'template_id' => { label => 'Host Template ID', default => '' }, - 'tree_id' => { label => 'Graph Tree ID', + 'tree_id' => { label => 'Graph Tree ID (optional)', + default => '' }, + 'description' => { label => 'Description (can use tokens $contact, $ip_addr and $description)', + default => 'Freeside $contact $description $ip_addr' }, + 'graphs_path' => { label => 'Graph Export Directory (user@host:/path/to/graphs/)', default => '' }, - 'description' => { label => 'Description (can use $ip_addr and $description tokens)', - default => 'Freeside $description $ip_addr' }, -# 'delete_graphs' => { label => 'Delete associated graphs and data sources when unprovisioning', -# type => 'checkbox', -# }, + 'import_freq' => { label => 'Minimum minutes between graph imports', + default => '5' }, + 'max_graph_size' => { label => 'Maximum size per graph (MB)', + default => '5' }, + 'delete_graphs' => { label => 'Delete associated graphs and data sources when unprovisioning', + type => 'checkbox', + }, + 'include_path' => { label => 'Path to cacti include dir (relative to script_path)', + default => '../site/include/' }, + 'cacti_graph_template_id' => { + 'label' => 'Graph Template', + 'type' => 'custom', + 'multiple' => 1, + }, + 'cacti_snmp_query_id' => { + 'label' => 'SNMP Query ID', + 'type' => 'custom', + 'multiple' => 1, + }, + 'cacti_snmp_query_type_id' => { + 'label' => 'SNMP Query Type ID', + 'type' => 'custom', + 'multiple' => 1, + }, + 'cacti_snmp_field' => { + 'label' => 'SNMP Field', + 'type' => 'custom', + 'multiple' => 1, + }, + 'cacti_snmp_value' => { + 'label' => 'SNMP Value', + 'type' => 'custom', + 'multiple' => 1, + }, ; %info = ( - 'svc' => 'svc_broadband', - 'desc' => 'Export service to cacti server, for svc_broadband services', - 'options' => \%options, - 'notes' => <<'END', + 'svc' => 'svc_broadband', + 'desc' => 'Export service to cacti server, for svc_broadband services', + 'post_config_element' => '/edit/elements/part_export/cacti.html', + 'options' => \%options, + 'notes' => <<'END', Add service to cacti upon provisioning, for broadband services.
-See FS::part_export::cacti documentation for details. +See documentation for details. END ); @@ -47,8 +103,23 @@ sub _export_insert { sub _export_delete { my ($self, $svc_broadband) = @_; + my $oldAutoCommit = $FS::UID::AutoCommit; + local $FS::UID::AutoCommit = 0; + my $dbh = dbh; + foreach my $page (qsearch('cacti_page',{ svcnum => $svc_broadband->svcnum })) { + my $error = $page->delete; + if ($error) { + $dbh->rollback if $oldAutoCommit; + return $error; + } + } my ($q,$error) = _delete_queue($self, $svc_broadband); - return $error; + if ($error) { + $dbh->rollback if $oldAutoCommit; + return $error; + } + $dbh->commit or die $dbh->errstr if $oldAutoCommit; + return ''; } sub _export_replace { @@ -103,7 +174,9 @@ sub _insert_queue { 'tree_id' => $self->option('tree_id'), 'description' => $self->option('description'), 'svc_desc' => $svc_broadband->description, + 'contact' => $svc_broadband->cust_main->contact, 'svcnum' => $svc_broadband->svcnum, + 'self' => $self ); return ($queue,$error); } @@ -119,7 +192,8 @@ sub _delete_queue { 'user' => $self->option('user'), 'hostname' => $svc_broadband->ip_addr, 'script_path' => $self->option('script_path'), -# 'delete_graphs' => $self->option('delete_graphs'), + 'delete_graphs' => $self->option('delete_graphs'), + 'include_path' => $self->option('include_path'), ); return ($queue,$error); } @@ -128,20 +202,25 @@ sub _delete_queue { sub ssh_insert { my %opt = @_; + my $self = $opt{'self'}; # Option validation die "Non-numerical Host Template ID, check export configuration\n" unless $opt{'template_id'} =~ /^\d+$/; die "Non-numerical Graph Tree ID, check export configuration\n" - unless $opt{'tree_id'} =~ /^\d+$/; + unless $opt{'tree_id'} =~ /^\d*$/; # Add host to cacti my $desc = $opt{'description'}; $desc =~ s/\$ip_addr/$opt{'hostname'}/g; $desc =~ s/\$description/$opt{'svc_desc'}/g; - $desc =~ s/'/'\\''/g; + $desc =~ s/\$contact/$opt{'contact'}/g; +#for some reason, device names with apostrophes fail to export graphs in Cacti +#just removing them for now, someday maybe dig to figure out why +# $desc =~ s/'/'\\''/g; + $desc =~ s/'//g; my $cmd = $php - . $opt{'script_path'} + . trailslash($opt{'script_path'}) . q(add_device.php --description=') . $desc . q(' --ip=') @@ -155,69 +234,105 @@ sub ssh_insert { my $id = $1; # Add host to tree + if ($opt{'tree_id'}) { + $cmd = $php + . trailslash($opt{'script_path'}) + . q(add_tree.php --type=node --node-type=host --tree-id=) + . $opt{'tree_id'} + . q( --host-id=) + . $id; + $response = ssh_cmd(%opt, 'command' => $cmd); + unless ( $response =~ /Added Node node-id: \((\d+)\)/ ) { + die "Host added, but error adding host to tree: $response"; + } + } + + # Get list of graph templates for new id $cmd = $php - . $opt{'script_path'} - . q(add_tree.php --type=node --node-type=host --tree-id=) - . $opt{'tree_id'} - . q( --host-id=) - . $id; - $response = ssh_cmd(%opt, 'command' => $cmd); - unless ( $response =~ /Added Node node-id: \((\d+)\)/ ) { - die "Error adding host to tree: $response"; + . trailslash($opt{'script_path'}) + . q(freeside_cacti.php --get-graph-templates --host-template=) + . $opt{'template_id'}; + $cmd .= q( --include-path=') . $self->option('include_path') . q(') + if $self->option('include_path'); + my $ginfo = { map { $_ ? ($_ => undef) : () } split(/\n/,ssh_cmd(%opt, 'command' => $cmd)) }; + + # Add extra config info + my @xtragid = split("\n", $self->option('cacti_graph_template_id')); + my @query_id = split("\n", $self->option('cacti_snmp_query_id')); + my @query_type_id = split("\n", $self->option('cacti_snmp_query_type_id')); + my @snmp_field = split("\n", $self->option('cacti_snmp_field')); + my @snmp_value = split("\n", $self->option('cacti_snmp_value')); + for (my $i = 0; $i < @xtragid; $i++) { + my $gtid = $xtragid[$i]; + $ginfo->{$gtid} ||= []; + push(@{$ginfo->{$gtid}},{ + 'gtid' => $gtid, + 'query_id' => $query_id[$i], + 'query_type_id' => $query_type_id[$i], + 'snmp_field' => $snmp_field[$i], + 'snmp_value' => $snmp_value[$i], + }); } - my $leaf_id = $1; - # Store id for generating graph urls - my $svc_broadband = qsearchs({ - 'table' => 'svc_broadband', - 'hashref' => { 'svcnum' => $opt{'svcnum'} }, - }); - die "Could not reload broadband service" unless $svc_broadband; - $svc_broadband->set('cacti_leaf_id',$leaf_id); - my $error = $svc_broadband->replace; - return $error if $error; - -# # Get list of graph templates for new id -# $cmd = $php -# . $opt{'script_path'} -# . q(freeside_cacti.php --get-graph-templates --host-template=) -# . $opt{'template_id'}; -# my @gtids = split(/\n/,ssh_cmd(%opt, 'command' => $cmd)); -# die "No graphs configured for host template" -# unless @gtids; -# -# # Create graphs -# foreach my $gtid (@gtids) { -# -# # sanity checks, should never happen -# next unless $gtid; -# die "Bad graph template: $gtid" -# unless $gtid =~ /^\d+$/; -# -# # create the graph -# $cmd = $php -# . $opt{'script_path'} -# . q(add_graphs.php --graph-type=cg --graph-template-id=) -# . $gtid -# . q( --host-id=) -# . $id; -# $response = ssh_cmd(%opt, 'command' => $cmd); -# die "Error creating graph $gtid: $response" -# unless $response =~ /Graph Added - graph-id: \((\d+)\)/; -# my $gid = $1; -# -# # add the graph to the tree -# $cmd = $php -# . $opt{'script_path'} -# . q(add_tree.php --type=node --node-type=graph --tree-id=) -# . $opt{'tree_id'} -# . q( --graph-id=) -# . $gid; -# $response = ssh_cmd(%opt, 'command' => $cmd); -# die "Error adding graph $gid to tree: $response" -# unless $response =~ /Added Node/; -# -# } #foreach $gtid + my @gdefs = map { + ref($ginfo->{$_}) ? @{$ginfo->{$_}} : {'gtid' => $_} + } keys %$ginfo; + warn "Host ".$opt{'hostname'}." exported to cacti, but no graphs configured" + unless @gdefs; + + # Create graphs + my $gerror = ''; + foreach my $gdef (@gdefs) { + # validate graph info + my $gtid = $gdef->{'gtid'}; + next unless $gtid; + $gerror .= " Bad graph template: $gtid" + unless $gtid =~ /^\d+$/; + my $isds = $gdef->{'query_id'} + || $gdef->{'query_type_id'} + || $gdef->{'snmp_field'} + || $gdef->{'snmp_value'}; + if ($isds) { + $gerror .= " Bad SNMP Query Id: " . $gdef->{'query_id'} + unless $gdef->{'query_id'} =~ /^\d+$/; + $gerror .= " Bad SNMP Query Type Id: " . $gdef->{'query_type_id'} + unless $gdef->{'query_type_id'} =~ /^\d+$/; + $gerror .= " SNMP Field cannot contain apostrophe" + if $gdef->{'snmp_field'} =~ /'/; + $gerror .= " SNMP Value cannot contain apostrophe" + if $gdef->{'snmp_value'} =~ /'/; + } + next if $gerror; + + # create the graph + $cmd = $php + . trailslash($opt{'script_path'}) + . q(add_graphs.php --graph-type=) + . ($isds ? 'ds' : 'cg') + . q( --graph-template-id=) + . $gtid + . q( --host-id=) + . $id; + if ($isds) { + $cmd .= q( --snmp-query-id=) + . $gdef->{'query_id'} + . q( --snmp-query-type-id=) + . $gdef->{'query_type_id'} + . q( --snmp-field=') + . $gdef->{'snmp_field'} + . q(' --snmp-value=') + . $gdef->{'snmp_value'} + . q('); + } + $response = ssh_cmd(%opt, 'command' => $cmd); + #might be more than one graph added, just testing success + $gerror .= "Error creating graph $gtid: $response" + unless $response =~ /Graph Added - graph-id: \((\d+)\)/; + + } #foreach $gtid + + # job fails, but partial export may have occurred + die $gerror . " Partial export occurred\n" if $gerror; return ''; } @@ -225,18 +340,256 @@ sub ssh_insert { sub ssh_delete { my %opt = @_; my $cmd = $php - . $opt{'script_path'} + . trailslash($opt{'script_path'}) . q(freeside_cacti.php --drop-device --ip=') . $opt{'hostname'} . q('); -# $cmd .= q( --delete-graphs) -# if $opt{'delete_graphs'}; + $cmd .= q( --delete-graphs) + if $opt{'delete_graphs'}; + $cmd .= q( --include-path=') . $opt{'include_path'} . q(') + if $opt{'include_path'}; my $response = ssh_cmd(%opt, 'command' => $cmd); die "Error removing from cacti: " . $response if $response; return ''; } +=head1 SUBROUTINES + +=over 4 + +=item process_graphs JOB PARAM + +Intended to be run as an FS::queue job. + +Copies graphs for a single service from Cacti export directory to FS cache, +generates basic html pages for this service with base64-encoded graphs embedded, +and stores the generated pages in the database. + +=back + +=cut + +sub process_graphs { + my ($job,$param) = @_; + + $job->update_statustext(10); + my $cachedir = trailslash($FS::UID::cache_dir,'cache.'.$FS::UID::datasrc,'cacti-graphs'); + + # load the service + my $svcnum = $param->{'svcnum'} || die "No svcnum specified"; + my $svc = qsearchs({ + 'table' => 'svc_broadband', + 'hashref' => { 'svcnum' => $svcnum }, + }) || die "Could not load svcnum $svcnum"; + + # load relevant FS::part_export::cacti object + my ($self) = $svc->cust_svc->part_svc->part_export('cacti'); + + $job->update_statustext(20); + + my $oldAutoCommit = $FS::UID::AutoCommit; + local $FS::UID::AutoCommit = 0; + my $dbh = dbh; + + # check for existing pages + my $now = time; + my %oldpages = map { ($_->graphnum || 'MAIN') => $_ } qsearch({ + 'table' => 'cacti_page', + 'hashref' => { 'svcnum' => $svcnum, 'exportnum' => $self->exportnum }, + 'select' => 'cacti_pagenum, exportnum, svcnum, graphnum, imported, thumbnail', #no need to load old content + 'order_by' => 'ORDER BY graphnum', + }); + + # if all existing pages are recent enough, do nothing and return + # (won't detect newly introduced graphs, but they can wait for next run) + my $uptodate = 0; + if (keys %oldpages) { + $uptodate = 1; + foreach my $oldpage (keys %oldpages) { + if ($oldpages{$oldpage}->imported <= $self->exptime($now)) { + $uptodate = 0; + last; + } + } + } + if ($uptodate) { + $job->update_statustext(100); + return ''; + } + + $job->update_statustext(30); + + # get list of graphs for this svc from cacti server + my $cmd = $php + . trailslash($self->option('script_path')) + . q(freeside_cacti.php --get-graphs --ip=') + . $svc->ip_addr + . q('); + $cmd .= q( --include-path=') . $self->option('include_path') . q(') + if $self->option('include_path'); + my @graphs = map { [ split(/\t/,$_) ] } + split(/\n/, ssh_cmd( + 'host' => $self->machine, + 'user' => $self->option('user'), + 'command' => $cmd + )); + + $job->update_statustext(40); + + # copy graphs from cacti server to cache + # requires version 2.6.4 of rsync, released March 2005 + my $rsync = File::Rsync->new({ + 'rsh' => 'ssh', + 'verbose' => 1, + 'recursive' => 1, + 'quote-src' => 1, + 'quote-dst' => 1, + 'source' => trailslash($self->option('graphs_path')), + 'dest' => $cachedir, + 'include' => [ + (map { q('**graph_).${$_}[0].q(*.png') } @graphs), + (map { q('**thumb_).${$_}[0].q(.png') } @graphs), + q('*/'), + q('- *'), + ], + }); + #don't know why a regular $rsync->exec isn't doing includes right, but this does + my $rscmd = join(' ',@{$rsync->getcmd()}); + my $error = system($rscmd); + die "rsync ($rscmd) failed with exit status $error" if $error; + + $job->update_statustext(50); + + # create html file contents + my $svchead = q() + . '

Service #' . $svcnum . '

' + . q(

Last updated ) . scalar(localtime($now)) . q(

); + my $svchtml = $svchead; + my $maxgraph = 1024 * 1024 * ($self->options('max_graph_size') || 5); + my $nographs = 1; + for (my $i = 0; $i <= $#graphs; $i++) { + my $graph = $graphs[$i]; + my $thumbfile = $cachedir . 'graphs/thumb_' . $$graph[0] . '.png'; + if (-e $thumbfile) { + if ( stat($thumbfile)->size() < $maxgraph ) { + $nographs = 0; + my $thumbnail = img_tag($thumbfile); + # add graph to main file + my $graphhead = q(

) . $$graph[1] . q(

); + $svchtml .= $graphhead; + $svchtml .= anchor_tag( $svcnum, $$graph[0], $thumbnail ); + # create graph details file + my $graphhtml = $svchead . $graphhead; + my $nodetail = 1; + my $j = 1; + # no easy way to tell what detail graphs should exist, + # and don't want detail graphs that are out of sync with thumbnail, + # so just use what we can find + while (-e (my $graphfile = $cachedir.'graphs/graph_'.$$graph[0].'_'.$j.'.png')) { + if ( stat($graphfile)->size() < $maxgraph ) { + $nodetail = 0; + $graphhtml .= img_tag($graphfile); + } + unlink($graphfile); + $j++; + } + $graphhtml .= '

No detail graphs to display for this graph

' + if $nodetail; + #delete old detail page + if ($oldpages{$$graph[0]}) { + $error = $oldpages{$$graph[0]}->delete; + if ($error) { + $dbh->rollback if $oldAutoCommit; + die $error; + } + } + #insert new detail page + my $newobj = new FS::cacti_page { + 'exportnum' => $self->exportnum, + 'svcnum' => $svcnum, + 'graphnum' => $$graph[0], + 'imported' => $now, + 'content' => $graphhtml, + 'thumbnail' => $thumbnail, + }; + $error = $newobj->insert; + if ($error) { + $dbh->rollback if $oldAutoCommit; + die $error; + } + } else { + $svchtml .= qq(

File $thumbfile is too large, skipping

); + } + unlink($thumbfile); + } else { + # try to use old page for this graph + if ($oldpages{$$graph[0]} && $oldpages{$$graph[0]}->thumbnail) { + $nographs = 0; + # add old graph to main file + my $graphhead = q(

) . $$graph[1] . q(

); + $svchtml .= $graphhead; + $svchtml .= qq(

Current graphs unavailable; using previously imported data.

); + $svchtml .= anchor_tag( $svcnum, $$graph[0], $oldpages{$$graph[0]}->thumbnail ); + } else { + $svchtml .= qq(

Error loading graph: $$graph[0]

); + } + } + # remove old page from hash even if it is being reused, + # remaining entries in hash will be deleted from database below + delete $oldpages{$$graph[0]} if $oldpages{$$graph[0]}; + $job->update_statustext(49 + int($i / @graphs) * 50); + } + $svchtml .= '

No graphs to display for this service

' + if $nographs; + # delete remaining old pages, including svc index + foreach my $oldpage (keys %oldpages) { + $error = $oldpages{$oldpage}->delete; + if ($error) { + $dbh->rollback if $oldAutoCommit; + die $error; + } + } + # insert new index page for svc + my $newobj = new FS::cacti_page { + 'exportnum' => $self->exportnum, + 'svcnum' => $svcnum, + 'graphnum' => '', + 'imported' => $now, + 'content' => $svchtml, + 'thumbnail' => '', + }; + $error = $newobj->insert; + if ($error) { + $dbh->rollback if $oldAutoCommit; + die $error; + } + + $dbh->commit or die $dbh->errstr if $oldAutoCommit; + + $job->update_statustext(100); + return ''; +} + +sub img_tag { + my $somefile = shift; + return q(
); +} + +sub anchor_tag { + my ($svcnum, $graphnum, $contents) = @_; + return q() + . $contents + . q(); +} + +#this gets used by everything else #fake false laziness, other ssh_cmds handle error/output differently sub ssh_cmd { use Net::OpenSSH; @@ -244,86 +597,75 @@ sub ssh_cmd { my $ssh = Net::OpenSSH->new($opt->{'user'}.'@'.$opt->{'host'}); die "Couldn't establish SSH connection: ". $ssh->error if $ssh->error; my ($output, $errput) = $ssh->capture2($opt->{'command'}); - die "Error running SSH command: ". $ssh->error if $ssh->error; + die "Error running SSH command: ". $opt->{'command'}. ' ERROR: ' . $ssh->error if $ssh->error; die $errput if $errput; return $output; } -=pod - -=head1 NAME - -FS::part_export::cacti - -=head1 SYNOPSIS - -Cacti integration for Freeside - -=head1 DESCRIPTION - -This module in particular handles FS::part_export object creation for Cacti integration; -consult any existing L documentation for details on how that works. -What follows is more general instructions for connecting your Cacti installation -to your Freeside installation. - -=head2 Connecting Cacti To Freeside - -Copy the freeside_cacti.php script from the bin directory of your Freeside -installation to the cli directory of your Cacti installation. Give this file -the same permissions as the other files in that directory, and create -(or choose an existing) user with sufficient permission to read these scripts. - -In the regular Cacti interface, create a Host Template to be used by -devices exported by Freeside, and note the template's id number. +#there's probably a better place to put this? +#makes sure there's a trailing slash between/after input +#doesn't add leading slashes +sub trailslash { + my @paths = @_; + my $out = ''; + foreach my $path (@paths) { + $out .= $path; + $out .= '/' unless $out =~ /\/$/; + } + return $out; +} -In Freeside, go to Configuration->Services->Provisioning exports to -add a new export. From the Add Export page, select cacti for Export then enter... +=head1 METHODS -* the User Name with permission to run scripts in the cli directory +=over 4 -* enter the full Script Path to that directory (eg /usr/share/cacti/cli/) +=item cleanup -* enter the Base Cacti URL for your cacti server (eg https://example.com/cacti/) +Removes all expired graphs for this export from the database. -* the Host Template ID for adding new devices +=cut -* the Graph Tree ID for adding new devices +sub cleanup { + my $self = shift; + my $oldAutoCommit = $FS::UID::AutoCommit; + local $FS::UID::AutoCommit = 0; + my $dbh = dbh; + my $sth = $dbh->prepare('DELETE FROM cacti_page WHERE exportnum = ? and imported <= ?') + or do { + $dbh->rollback if $oldAutoCommit; + return $dbh->errstr; + }; + $sth->execute($self->exportnum,$self->exptime) + or do { + $dbh->rollback if $oldAutoCommit; + return $dbh->errstr; + }; + $dbh->commit or return $dbh->errstr if $oldAutoCommit; + return ''; +} -* the Description for new devices; you can use the tokens - $ip_addr and $description to include the equivalent fields - from the broadband service definition +=item exptime [ TIME ] -After adding the export, go to Configuration->Services->Service definitions. -The export you just created will be available for selection when adding or -editing broadband service definitions. +Accepts optional current time, defaults to actual current time. -When properly configured broadband services are provisioned, they should now -be added to Cacti using the Host Template you specified, and the created device -will also be added to the specified Graph Tree. +Returns timestamp for the oldest possible non-expired graph import, +based on the import_freq option. -Once added, a link to the graphs for this host will be available when viewing -the details of the provisioned service in Freeside (you will need to authenticate -into Cacti to view them.) +=cut -Devices will be deleted from Cacti when the service is unprovisioned in Freeside, -and they will be deleted and re-added if the ip address changes. +sub exptime { + my $self = shift; + my $now = shift || time; + return $now - 60 * ($self->option('import_freq') || 5); +} -Currently, graphs themselves must still be added in cacti by hand or some -other form of automation tailored to your specific graph inputs and data sources. +=back =head1 AUTHOR Jonathan Prykop jonathan@freeside.biz -=head1 LICENSE AND COPYRIGHT - -Copyright 2015 Freeside Internet Services - -This program is free software; you can redistribute it and/or | -modify it under the terms of the GNU General Public License | -as published by the Free Software Foundation. - =cut 1;