From 233dcab1ee337c6ab35abde5c3f22d57b40eb82c Mon Sep 17 00:00:00 2001 From: Mark Wells Date: Tue, 23 Jul 2013 10:56:21 -0700 Subject: [PATCH] VoIP Innovations CDR format and import script, #23646 --- FS/FS/cdr/voip_innovations.pm | 70 ++++++++++++++++++++++ bin/cdr-voip_innovations.import | 127 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+) create mode 100644 FS/FS/cdr/voip_innovations.pm create mode 100644 bin/cdr-voip_innovations.import diff --git a/FS/FS/cdr/voip_innovations.pm b/FS/FS/cdr/voip_innovations.pm new file mode 100644 index 000000000..29dffea37 --- /dev/null +++ b/FS/FS/cdr/voip_innovations.pm @@ -0,0 +1,70 @@ +package FS::cdr::voip_innovations; + +use strict; +use base qw( FS::cdr ); +use vars qw( %info ); +use FS::cdr qw( _cdr_date_parser_maker _cdr_min_parser_maker ); + +%info = ( + 'name' => 'VoIP Innovations', + 'weight' => 540, + 'header' => 1, + 'type' => 'csv', + 'sep_char' => ';', + + 'import_fields' => [ + # CallType + sub { my($cdr, $type) = @_; + # DNIS and ANI are the same columns regardless of direction, + # so all we need to assign is the IP address + if ($type =~ /^term/i) { + $cdr->dst_ip_addr( $cdr->get('ipaddr') ); + } else { + $cdr->src_ip_addr( $cdr->get('ipaddr') ); + } + # also strip the leading '1' or '+1' from src/dst + foreach (qw(src dst)) { + my $num = $cdr->get($_); + $num =~ s/^\+?1//; + $cdr->set($_, $num); + } + }, + # StartTime + _cdr_date_parser_maker('startdate'), + # StopTime + _cdr_date_parser_maker('enddate'), + # CallDuration + sub { my($cdr, $duration) = @_; + $cdr->duration(sprintf('%.0f',$duration)); + }, + # BillDuration (granularized) + sub { my($cdr, $billsec) = @_; + $cdr->billsec(sprintf('%.0f',$billsec)); + }, + # CallMinimum, CallIncrement (used to granularize BillDuration) + '', '', + # BasePrice + '', + # CallPrice + 'upstream_price', + # TransactionId (seems not to be meaningful to us) + '', + # CustomerIP + 'ipaddr', # will go to src or dst addr, depending + # ANI + 'src', + # ANIState + '', # would be really useful for classifying inter/intrastate calls, + # except we don't use it + # DNIS + 'dst', + # LRN (Local Routing Number of the destination) + '', + # DNISState, DNISLATA, DNISOCN + '', '', '', + # OrigTier, TermRateDeck + '', '', # these are upstream rate plans, but they're not numeric + ], +); + +1; diff --git a/bin/cdr-voip_innovations.import b/bin/cdr-voip_innovations.import new file mode 100644 index 000000000..6219829a4 --- /dev/null +++ b/bin/cdr-voip_innovations.import @@ -0,0 +1,127 @@ +#!/usr/bin/perl + +use strict; +use Getopt::Std; +use Date::Format; +use File::Temp 'tempdir'; +use Net::FTP; +use FS::UID qw(adminsuidsetup datasrc dbh); +use FS::cdr; +use FS::cdr_batch; +use FS::Record qw(qsearch qsearchs); + + +### +# parse command line +### + +use vars qw( $opt_d $opt_v $opt_c $opt_s $opt_e $opt_a ); +getopts('dvc:s:e:a'); + +my ($user, $login, $password) = @ARGV; +$user and $login and $password or die &usage; + +my $dbh = adminsuidsetup $user; +$FS::UID::AutoCommit = 0; + +# index already-downloaded batches +my @previous = qsearch({ + 'table' => 'cdr_batch', + 'hashref' => { 'cdrbatch' => {op=>'like', value=>'voip_innovations%'} }, + 'order_by' => 'ORDER BY cdrbatch DESC', +}); +my %exists = map {$_->cdrbatch => 1} @previous; + +my $tempdir = tempdir( CLEANUP => !$opt_v ); + +my $format = 'voip_innovations'; +my $hostname = 'cdrs.globalpopsvoip.com'; + +my $ftp = Net::FTP->new($hostname, Debug => $opt_d) + or die "Can't connect to $hostname: $@\n"; + +$ftp->login($login, $password) + or die "Login failed: ".$ftp->message."\n"; + +### +# get the file list +### + +warn "Retrieving directory listing\n" if $opt_v; + +$ftp->cwd('/'); +my @dirs = $ftp->ls(); +warn scalar(@dirs)." directories found.\n" if $opt_v; +# apply date range +if ( $opt_a ) { + my $most_recent = $previous[0]; + if ($most_recent) { + if ($most_recent->cdrbatch =~ /^voip_innovations-(\d+)/) { + my $date = $1; + warn "limiting to dates > $date (from most recent batch)\n" if $opt_v; + @dirs = grep {$_ > $date} @dirs; + } + } # else download them all +} +if ( $opt_s ) { + # start date + # normalize date format + $opt_s = str2time('%Y%m%d', time2str($opt_s)) if $opt_s =~ /\D/; + warn "limiting to dates > $opt_s\n" if $opt_v; + @dirs = grep {$_ > $opt_s} @dirs; +} +if ( $opt_e ) { + # end date + $opt_e = str2time('%Y%m%d', time2str($opt_e)) if $opt_e =~ /\D/; + warn "limiting to dates < $opt_e\n" if $opt_v; + @dirs = grep {$_ < $opt_e} @dirs; +} +warn scalar(@dirs) ." to be downloaded\n" if $opt_v; +foreach my $dir (@dirs) { + $ftp->cwd($dir); + foreach my $file ($ftp->ls) { + warn "downloading $file\n" if $opt_v; + $ftp->get($file, "$tempdir/$file"); + warn "processing $file\n" if $opt_v; + + # "voip_innovations-20130628/20130628_20130628.CDR" + my $batchname = "$format-$dir/$file"; + if ($exists{$batchname}) { + warn "already imported $file\n"; + next; + } + my $import_options = { + 'file' => "$tempdir/$file", + 'format' => $format, + 'batch_namevalue' => $batchname, + 'empty_ok' => 1, + }; + $import_options->{'cdrtypenum'} = $opt_c if $opt_c; + + my $error = FS::cdr::batch_import($import_options); + + if ( $error ) { + die "error processing $dir/$file: $error\n"; + } + } + $ftp->cwd('..'); +} +warn "finished\n"; +$dbh->commit; + +### +# subs +### + +sub usage { + "Usage: \n cdr-voip_innovations.import user login password\n [ options ] + Options: + -v: be verbose + -d: enable FTP debugging (very noisy) + -c num: apply a cdrtypenum to the imported CDRs + -s date: start date + -e date: end date + -a: automatically choose start date from most recently downloaded batch + "; +} + -- 2.11.0