#!@PERL@ # Copyright (C) 2002 Stanislav Sinyagin # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. # $Id: devdiscover.in,v 1.1 2010-12-27 00:04:02 ivan Exp $ # Stanislav Sinyagin # Collect the router information and create the XML file BEGIN { require '@devdiscover_config_pl@'; } use strict; use Getopt::Long; use XML::LibXML; use Torrus::Log; use Torrus::DevDiscover; use Torrus::ConfigBuilder; $| = 1; my @infiles; my $makedirs; my $limitre; my $forcebundle; my $fallback; my $workerThreads = 0; # Hidden parameter for debugging my $snmpdebug = 0; my $debug = 0; my $verbose = 0; my %formatsSupported = ( '1.0' => 1 ); my $creator = "Torrus version @VERSION@\n" . "This file was generated by command:\n" . $0 . " \\\n"; foreach my $arg ( @ARGV ) { if( $arg =~ /^--/ ) { $creator .= ' ' . $arg . ' '; } else { $creator .= "\'" . $arg . "\'\\\n"; } } $creator .= "\n On " . scalar(localtime(time)); my $ok = GetOptions( 'in=s' => \@infiles, 'mkdir' => \$makedirs, 'limit=s' => \$limitre, 'forcebundle' => \$forcebundle, 'fallback=i' => \$fallback, 'threads=i' => \$workerThreads, 'snmpdebug' => \$snmpdebug, 'verbose' => \$verbose, 'debug' => \$debug ); if( $ok and scalar( @ARGV ) > 0 ) { push( @infiles, @ARGV ); } if( not $ok or scalar(@infiles) == 0 or ($workerThreads > 1 and not $Torrus::Global::threadsEnabled ) ) { print STDERR "Usage: $0 --in=filename.ddx options... [ddx files]\n", "Options:\n", " --in=filename.ddx discovery instructions XML file(s)\n", " --mkdir create data-dir directories\n", " --limit=regexp limit the discovery by output files\n", " --forcebundle always write the bundle file\n", " --fallback=integer maximum age of XML file to fall back to\n", " --threads=integer number of parallel discovery threads\n", " --verbose print extra information\n", " --debug print debugging information\n", " --snmpdebug print SNMP protocol details\n", "\n"; if( not $Torrus::Global::threadsEnabled ) { print STDERR "Multithreading is NOT SUPPORTED by current " . "perl interpreter\n"; } exit 1; } if( $snmpdebug ) { $Net::SNMP::Transport::UDP::DEBUG = 1; $Net::SNMP::Message::DEBUG = 1; $Net::SNMP::MessageProcessing::DEBUG = 1; $Net::SNMP::Dispatcher::DEBUG = 1; } if( $debug ) { Torrus::Log::setLevel('debug'); } elsif( $verbose ) { Torrus::Log::setLevel('verbose'); } my $everythingsOk = 1; my $perOutfileHostParams = {}; my %outputBundles; foreach my $infile ( @infiles ) { if( not -r $infile ) { my $altfile = $Torrus::Global::discoveryDir . $infile; if( not -r $altfile ) { Error('Cannot find file ' . $infile . ' neither in current directory nor in ' . $Torrus::Global::discoveryDir); exit 1; } else { $infile = $altfile; } } Verbose('Processing ' . $infile); my $parser = new XML::LibXML; my $doc; eval { $doc = $parser->parse_file( $infile ); }; if( $@ ) { Error("Failed to parse $infile: $@"); exit 1; } my $root = $doc->documentElement(); if( $root->nodeName() ne 'snmp-discovery' ) { Error('XML root element is not "snmp-discovery" in ' . $infile); exit 1; } my $format_version = (($root->getElementsByTagName('file-info'))[0]-> getElementsByTagName('format-version'))[0]->textContent(); $format_version =~ s/\s//g; if( not $format_version or not $formatsSupported{$format_version} ) { Error('Invalid format or format version not supported: ' . $infile); exit 1; } my $globalParams = parseParams( $root ); # Parse the body of the XML foreach my $hostNode ( $root->getChildrenByTagName('host') ) { my $hostParams = parseParams( $hostNode, $globalParams ); normalizeParams( $hostParams ); my $outfile = $hostParams->{'output-file'}; if( not exists($perOutfileHostParams->{$outfile}) ) { $perOutfileHostParams->{$outfile} = []; } push( @{$perOutfileHostParams->{$outfile}}, $hostParams ); my $outBundles = $hostParams->{'output-bundle'}; if( length( $outBundles ) > 0 ) { foreach my $bundleName ( split( /\s*,\s*/, $outBundles ) ) { $bundleName = absXmlFilename( $bundleName ); $outputBundles{$bundleName}{ relXmlFilename($outfile) } = 1; } } } } # Start discovery my $jobQueue; my $bundleDeletionQueue; my $confBuildSemaphore; if( $workerThreads > 1 ) { require threads; require threads::shared; require Thread::Queue; require Thread::Semaphore; threads::shared::share( \$everythingsOk ); $jobQueue = new Thread::Queue; $bundleDeletionQueue = new Thread::Queue; $confBuildSemaphore = new Thread::Semaphore; # Enqueue the output filenames foreach my $outfile ( sort keys %{$perOutfileHostParams} ) { if( not matchLimitRe( $outfile ) ) { next; } $jobQueue->enqueue( $outfile ); } # Start the worker threads my @workers; foreach my $i ( 1..$workerThreads ) { push( @workers, threads->create( \&discoveryThread ) ); } # Wait for workers to finish the jobs while( my $thr = shift( @workers ) ) { my $tid = $thr->tid(); $thr->join(); Debug('Cleaning up thread #' . $tid); undef $thr; } # Process the files to be excluded from bundles if( not $everythingsOk ) { my $outfile; while( defined( $outfile = $bundleDeletionQueue->dequeue_nb() ) ) { removeFromBundle( $outfile ); } } } else { # Single-thread operation foreach my $outfile ( sort keys %{$perOutfileHostParams} ) { if( not matchLimitRe( $outfile ) ) { next; } if( not doDiscover( $outfile ) ) { removeFromBundle( $outfile ); } } } # Discovery finished, do the bundles if( scalar( keys %outputBundles ) > 0 ) { if( defined( $limitre ) ) { Warn('Cannot write bundles with --limit option specified. ' . 'Bundle files remain unchanged'); } elsif( $everythingsOk ) { foreach my $bundleName ( sort keys %outputBundles ) { my $cb = new Torrus::ConfigBuilder; $cb->addCreatorInfo( $creator ); foreach my $bundleMember ( sort keys %{$outputBundles{$bundleName}} ) { $cb->addFileInclusion( $bundleMember ); } my $ok = $cb->toFile( $bundleName ); if( $ok ) { Verbose('Wrote bundle to ' . $bundleName); } else { Error('Cannot write bundle to ' . $bundleName . ': ' . $!); $everythingsOk = 0; } } } else { Error('Skipping bundles generation because of errors'); } } exit($everythingsOk ? 0:1); sub parseParams { my $parentNode = shift; my $paramhash = shift; # Clone the parameters hash my $ret = {}; if( $paramhash ) { while( my($key, $val) = each %{$paramhash} ) { $ret->{$key} = $val; } } foreach my $paramNode ( $parentNode->getChildrenByTagName('param') ) { my $param = $paramNode->getAttribute('name'); my $value = $paramNode->getAttribute('value'); if( not $param ) { Error("Parameter without name"); exit 1; } if( not defined( $value ) ) { $value = $paramNode->textContent(); } # Remove spaces in the head and tail. $value =~ s/^\s+//; $value =~ s/\s+$//; $ret->{$param} = $value; } return $ret; } sub normalizeParams { my $params = shift; if( not defined( $params->{'output-file'} ) ) { Warn('output-file parameter is not defined. Using routers.xml'); $params->{'output-file'} = 'routers.xml'; } else { $params->{'output-file'} = absXmlFilename( $params->{'output-file'} ); } if( defined( $params->{'host-subtree'} ) ) { my $subtree = $params->{'host-subtree'}; if( $subtree !~ /^\/[0-9A-Za-z_\-\.\/]*$/ or $subtree =~ /\.\./ ) { Error("Invalid format for subtree name: " . $subtree); exit 1; } } if( defined( $params->{'snmp-community'} ) ) { # Remove any possible Unicode character treatment $params->{'snmp-community'} = pack( 'A*', $params->{'snmp-community'} ); } } # Replaces $XMLCONFIG with the XML root directory sub absXmlFilename { my $filename = shift; my $subst = '$XMLCONFIG'; my $offset = index( $filename, $subst ); if( $offset >= 0 ) { my $len = length( $subst ); substr( $filename, $offset, $len ) = $Torrus::Global::siteXmlDir; } else { if( $filename !~ /^\// ) { $filename = $Torrus::Global::siteXmlDir . '/' . $filename; } } return $filename; } # Removes XML root directory from path sub relXmlFilename { my $filename = shift; my $subst = $Torrus::Global::siteXmlDir; my $len = length( $subst ); if( $filename =~ /^\// ) { my $offset = index( $filename, $subst ); if( $offset == 0 ) { $filename = substr( $filename, $len ); # we don't know if xmldir has a trailing slash $filename =~ s/^\///; } } return $filename; } sub matchLimitRe { my $filename = shift; if( defined( $limitre ) ) { $filename =~ s/^.*\///; if( $filename !~ $limitre ) { return 0; } } return 1; } # Pick up next available outfile until the job queue is empty sub discoveryThread { Torrus::Log::setTID( threads->tid() ); Debug('Started thread #' . threads->tid()); my $outfile; while( defined( $outfile = $jobQueue->dequeue_nb() )) { if( not doDiscover( $outfile ) ) { $bundleDeletionQueue->enqueue( $outfile ); } } Debug('Finished thread #' . threads->tid()); } sub doDiscover { my $outfile = shift; Verbose('Preparing to write ' . $outfile); my $dd = new Torrus::DevDiscover; my $ok = 1; foreach my $hostParams ( @{$perOutfileHostParams->{$outfile}} ) { $ok = $dd->discover( $hostParams ); if( not $ok ) { Error($outfile . ' was not written because of errors'); $everythingsOk = 0; last; } } if( $ok ) { # LibXML2 is not thread-safe, so we create the XML files # one at a time if( $workerThreads > 1 ) { $confBuildSemaphore->down(); } my $cb = new Torrus::ConfigBuilder; $cb->addCreatorInfo( $creator ); $dd->buildConfig( $cb ); $cb->addRequiredFiles(); $cb->addStatistics(); $ok = $cb->toFile( $outfile ); if( $ok ) { Verbose('Wrote ' . $outfile); } else { Error('Cannot write ' . $outfile . ': ' . $!); $everythingsOk = 0; } if( $workerThreads > 1 ) { $confBuildSemaphore->up(); } } if( $makedirs ) { if( $everythingsOk ) { # Not sure if these calls are reentrant if( $workerThreads > 1 ) { $confBuildSemaphore->down(); } my ($login,$pass,$uid,$gid) = getpwnam('@torrus_user@') or die "Cannot get user details for @torrus_user@"; foreach my $dir ( $dd->listDataDirs() ) { if( not -d $dir ) { Debug('Creating directory: ' . $dir); mkdir( $dir ) or Error('Cannot create directory: ' . $dir . ': ' . $!); chown( $uid, $gid, $dir ) or Error('Cannot change ownership for ' . $dir . ': ' . $!); chmod( 02755, $dir ) or Error('Cannot chmod 02755 for ' . $dir . ': ' . $!); } } if( $workerThreads > 1 ) { $confBuildSemaphore->up(); } } else { Error('Skipping mkdir because of errors'); } } return $ok; } sub removeFromBundle { my $outfile = shift; my $relname = relXmlFilename($outfile); my $removeFromBundle = 1; if( $forcebundle ) { if( defined( $fallback ) and -e $outfile and -M $outfile <= $fallback ) { Warn('Falling back to the old version of ' . $relname); $removeFromBundle = 0; } $everythingsOk = 1; } if( $removeFromBundle ) { foreach my $bundleName ( sort keys %outputBundles ) { if( exists( $outputBundles{$bundleName}{$relname} ) ) { delete $outputBundles{$bundleName}{$relname}; Warn('Bundle ' . $bundleName . ' will not have ' . $relname . ' included because of errors'); } } } } # Local Variables: # mode: perl # indent-tabs-mode: nil # perl-indent-level: 4 # End: