VoIP Innovations CDR format and import script, #23646

diff --git a/FS/FS/cdr/voip_innovations.pm b/FS/FS/cdr/voip_innovations.pm
new file mode 100644 (file)
index 0000000..29dffea
--- /dev/null
+++ b/FS/FS/cdr/voip_innovations.pm
@@ -0,0 +1,70 @@
+package FS::cdr::voip_innovations;
+
+use strict;
+use base qw( FS::cdr );
+use vars qw( %info );
+use FS::cdr qw( _cdr_date_parser_maker _cdr_min_parser_maker );
+
+%info = (
+  'name'          => 'VoIP Innovations',
+  'weight'        => 540,
+  'header'        => 1,
+  'type'          => 'csv',
+  'sep_char'      => ';',
+
+  'import_fields' => [
+    # CallType
+    sub { my($cdr, $type) = @_;
+          # DNIS and ANI are the same columns regardless of direction,
+          # so all we need to assign is the IP address
+          if ($type =~ /^term/i) {
+            $cdr->dst_ip_addr( $cdr->get('ipaddr') );
+          } else {
+            $cdr->src_ip_addr( $cdr->get('ipaddr') );
+          }
+          # also strip the leading '1' or '+1' from src/dst
+          foreach (qw(src dst)) {
+            my $num = $cdr->get($_);
+            $num =~ s/^\+?1//;
+            $cdr->set($_, $num);
+          }
+        },
+    # StartTime
+    _cdr_date_parser_maker('startdate'),
+    # StopTime
+    _cdr_date_parser_maker('enddate'),
+    # CallDuration
+    sub { my($cdr, $duration) = @_;
+          $cdr->duration(sprintf('%.0f',$duration));
+        },
+    # BillDuration (granularized)
+    sub { my($cdr, $billsec) = @_;
+          $cdr->billsec(sprintf('%.0f',$billsec));
+        },
+    # CallMinimum, CallIncrement (used to granularize BillDuration)
+    '', '',
+    # BasePrice
+    '',
+    # CallPrice
+    'upstream_price',
+    # TransactionId (seems not to be meaningful to us)
+    '',
+    # CustomerIP
+    'ipaddr', # will go to src or dst addr, depending
+    # ANI
+    'src',
+    # ANIState
+    '', # would be really useful for classifying inter/intrastate calls, 
+        # except we don't use it
+    # DNIS
+    'dst',
+    # LRN (Local Routing Number of the destination)
+    '',
+    # DNISState, DNISLATA, DNISOCN
+    '', '', '',
+    # OrigTier, TermRateDeck
+    '', '', # these are upstream rate plans, but they're not numeric
+  ],
+);
+
+1;

diff --git a/bin/cdr-voip_innovations.import b/bin/cdr-voip_innovations.import
new file mode 100644 (file)
index 0000000..6219829
--- /dev/null
+++ b/bin/cdr-voip_innovations.import
@@ -0,0 +1,127 @@
+#!/usr/bin/perl
+
+use strict;
+use Getopt::Std;
+use Date::Format;
+use File::Temp 'tempdir';
+use Net::FTP;
+use FS::UID qw(adminsuidsetup datasrc dbh);
+use FS::cdr;
+use FS::cdr_batch;
+use FS::Record qw(qsearch qsearchs);
+
+
+###
+# parse command line
+###
+
+use vars qw( $opt_d $opt_v $opt_c $opt_s $opt_e $opt_a );
+getopts('dvc:s:e:a');
+
+my ($user, $login, $password) = @ARGV;
+$user and $login and $password or die &usage;
+
+my $dbh = adminsuidsetup $user;
+$FS::UID::AutoCommit = 0;
+
+# index already-downloaded batches
+my @previous = qsearch({
+    'table'     => 'cdr_batch',
+    'hashref'   => { 'cdrbatch' => {op=>'like', value=>'voip_innovations%'} },
+    'order_by'  => 'ORDER BY cdrbatch DESC',
+});
+my %exists = map {$_->cdrbatch => 1} @previous;
+
+my $tempdir = tempdir( CLEANUP => !$opt_v );
+
+my $format = 'voip_innovations';
+my $hostname = 'cdrs.globalpopsvoip.com';
+
+my $ftp = Net::FTP->new($hostname, Debug => $opt_d)
+  or die "Can't connect to $hostname: $@\n";
+
+$ftp->login($login, $password)
+  or die "Login failed: ".$ftp->message."\n";
+
+###
+# get the file list
+###
+
+warn "Retrieving directory listing\n" if $opt_v;
+
+$ftp->cwd('/');
+my @dirs = $ftp->ls();
+warn scalar(@dirs)." directories found.\n" if $opt_v;
+# apply date range
+if ( $opt_a ) {
+  my $most_recent = $previous[0];
+  if ($most_recent) {
+    if ($most_recent->cdrbatch =~ /^voip_innovations-(\d+)/) {
+      my $date = $1;
+      warn "limiting to dates > $date (from most recent batch)\n" if $opt_v;
+      @dirs = grep {$_ > $date} @dirs;
+    }
+  } # else download them all
+}
+if ( $opt_s ) {
+  # start date
+  # normalize date format
+  $opt_s = str2time('%Y%m%d', time2str($opt_s)) if $opt_s =~ /\D/;
+  warn "limiting to dates > $opt_s\n" if $opt_v;
+  @dirs = grep {$_ > $opt_s} @dirs;
+}
+if ( $opt_e ) {
+  # end date
+  $opt_e = str2time('%Y%m%d', time2str($opt_e)) if $opt_e =~ /\D/;
+  warn "limiting to dates < $opt_e\n" if $opt_v;
+  @dirs = grep {$_ < $opt_e} @dirs;
+}
+warn scalar(@dirs) ." to be downloaded\n" if $opt_v;
+foreach my $dir (@dirs) {
+  $ftp->cwd($dir);
+  foreach my $file ($ftp->ls) {
+    warn "downloading $file\n" if $opt_v;
+    $ftp->get($file, "$tempdir/$file");
+    warn "processing $file\n" if $opt_v;
+
+    # "voip_innovations-20130628/20130628_20130628.CDR"
+    my $batchname = "$format-$dir/$file";
+    if ($exists{$batchname}) {
+      warn "already imported $file\n";
+      next;
+    }
+    my $import_options = {
+      'file'            => "$tempdir/$file",
+      'format'          => $format,
+      'batch_namevalue' => $batchname,
+      'empty_ok'        => 1,
+    };
+    $import_options->{'cdrtypenum'} = $opt_c if $opt_c;
+  
+    my $error = FS::cdr::batch_import($import_options);
+
+    if ( $error ) {
+      die "error processing $dir/$file: $error\n";
+    }
+  }
+  $ftp->cwd('..');
+}
+warn "finished\n";
+$dbh->commit;
+
+###
+# subs
+###
+
+sub usage {
+  "Usage: \n  cdr-voip_innovations.import user login password\n [ options ]
+  Options:
+    -v: be verbose
+    -d: enable FTP debugging (very noisy)
+    -c num: apply a cdrtypenum to the imported CDRs
+    -s date: start date
+    -e date: end date
+    -a: automatically choose start date from most recently downloaded batch
+  ";
+}
+