diff options
author | Ivan Kohler <ivan@freeside.biz> | 2018-02-09 19:10:00 -0800 |
---|---|---|
committer | Ivan Kohler <ivan@freeside.biz> | 2018-02-09 19:10:00 -0800 |
commit | d45dd4a826f314fb5459747590d3e11cd80c211f (patch) | |
tree | c1dd2edd4bc42b12cc9a995e95dd7fb630da925e /FS/bin | |
parent | 4b67c9f8cfc9f944b7758e7e69ac1f9f188ffa47 (diff) | |
parent | 15d596e3090f3bde642917b56563736cd1ee2e90 (diff) |
Merge branch 'master' of git.freeside.biz:/home/git/freeside
Diffstat (limited to 'FS/bin')
-rwxr-xr-x | FS/bin/freeside-cdr-aninetworks-import | 222 | ||||
-rw-r--r-- | FS/bin/freeside-cdr-freeswitch | 11 |
2 files changed, 228 insertions, 5 deletions
diff --git a/FS/bin/freeside-cdr-aninetworks-import b/FS/bin/freeside-cdr-aninetworks-import new file mode 100755 index 0000000..b5fc226 --- /dev/null +++ b/FS/bin/freeside-cdr-aninetworks-import @@ -0,0 +1,222 @@ +#!/usr/bin/perl + +use strict; +use Getopt::Std; +use Date::Format; +use File::Temp 'tempdir'; +use Net::SFTP::Foreign::Compat; +use FS::UID qw(adminsuidsetup datasrc dbh); +use FS::cdr; +use FS::cdr_batch; +use FS::Record qw(qsearch qsearchs); +use Date::Format 'time2str'; +use Date::Parse 'str2time'; + + +### +# parse command line +### + +use vars qw( $opt_d $opt_v $opt_c $opt_s $opt_e $opt_a ); +getopts('dvc:s:e:a'); + +my ($user, $login, $password) = @ARGV; +($user and $login and $password) or die &usage; + +my $dbh = adminsuidsetup $user; +$FS::UID::AutoCommit = 0; + +# index already-downloaded batches +my @previous = qsearch({ + 'table' => 'cdr_batch', + 'hashref' => { 'cdrbatch' => {op=>'like', value=>'ani_networks%'} }, + 'order_by' => 'ORDER BY cdrbatch DESC', +}); +my %exists = map {$_->cdrbatch => 1} @previous; + +my $format = 'ani_networks'; +my $host = 'arkftp.aninetworks.com'; + +### +# get the file list +### + +warn "Retrieving directory listing\n" if $opt_v; + +my $sftp = sftp(); + +## get the current working dir +my $cwd = $sftp->cwd; + +## switch to CDR dir +$sftp->setcwd($cwd . '/CDR') or die "can't chdir to $cwd/CDR\n"; + +my $ls = $sftp->ls('.', wanted => qr/^UYM.*.zip$/i, names_only =>1 ); +my @files = @$ls; + +warn scalar(@files)." CDR files found.\n" if $opt_v; +# apply date range from last downloaded batch. +if ( $opt_a ) { + my $most_recent = $previous[0]; + if ($most_recent) { + if ($most_recent->cdrbatch =~ /^*Daily_(\d+)_/) { + my $date = $1; + warn "limiting to dates >= $date (from most recent batch)\n" if $opt_v; + @files = grep { /^*Daily_(\d+)_/ && $1 >= $date } @files; + } + } +} + +# apply a start date if given +if ( $opt_s ) { + # normalize date format + $opt_s = time2str('%Y%m%d', str2time($opt_s)) if $opt_s =~ /\D/; + warn "limiting to dates > $opt_s\n" if $opt_v; + @files= grep { /^*Daily_(\d+)_/ && $1 >= $opt_s } @files; +} + +# apply a end date if given +if ( $opt_e ) { + # normalize date format + $opt_e = time2str('%Y%m%d', str2time($opt_e)) if $opt_e =~ /\D/; + warn "limiting to dates < $opt_e\n" if $opt_v; + @files= grep { /^*Daily_(\d+)_/ && $1 < $opt_e } @files; +} + +warn scalar(@files) ." files to be downloaded\n" if $opt_v; +foreach my $file (@files) { + + my $tmpdir = tempdir( CLEANUP => $opt_v ); + + warn "downloading $file to $tmpdir\n" if $opt_v; + $sftp = sftp(); + $sftp->get($file, "$tmpdir/$file"); + + ## extract zip file + if(system ("unzip $tmpdir/$file -d $tmpdir") != 0) { + unlink "$tmpdir/$file"; + my $error = "unzip of '$tmpdir/$file' failed\n"; + if ( $opt_s ) { + warn $error; + next; + } else { + die $error; + } + } + + warn "processing $file\n" if $opt_v; + + my $batchname = "$format-$file"; + if ($exists{$batchname}) { + warn "already imported $file\n"; + next; + } + + my $unzipped_file = $file; + $unzipped_file =~ s/.zip/.txt/i; + + warn "going to import file $unzipped_file" if $opt_v; + + my $import_options = { + 'file' => "$tmpdir/$unzipped_file", + 'format' => $format, + 'batch_namevalue' => $batchname, + 'empty_ok' => 1, + }; + $import_options->{'cdrtypenum'} = $opt_c if $opt_c; + + my $error = FS::cdr::batch_import($import_options); + + if ( $error ) { + die "error processing $unzipped_file: $error\n"; + } +} +warn "finished\n" if $opt_v; +$dbh->commit; + +### +# subs +### + +sub usage { + "Usage: \n freeside-cdr-aninetworks-import [ options ] user login password + Options: + -v: be verbose + -d: enable FTP debugging (very noisy) + -c num: apply a cdrtypenum to the imported CDRs + -s date: start date + -e date: end date + -a: automatically choose start date from most recently downloaded batch + +"; +} + +sub sftp { + + #reuse connections + return $sftp if $sftp && $sftp->cwd; + + my %sftp = ( host => $host, + user => $login, + password => $password, + more => [-o => 'StrictHostKeyChecking no'], + ); + + $sftp = Net::SFTP::Foreign->new(%sftp); + $sftp->error and die "SFTP connection failed: ". $sftp->error; + + $sftp; +} + +=head1 NAME + +freeside-cdr-aninetworks-import - Download CDR files from a remote server via SFTP + +=head1 SYNOPSIS + + freeside-cdr-aninetworks-import [ -v ] [ -d ] [ -a ] [ -c cdrtypenum ] + [ -s startdate ] [ -e enddate ] user sftpuser sftppassword + +=head1 DESCRIPTION + +Command line tool to download CDR files from a remote server via SFTP +and then import them into the database. + +-v: be verbose + +-d: enable sftp debugging (very noisy) + +-a: automatically choose start date from most recently downloaded batch + +-c: cdrtypenum to set, defaults to none + +-s: if specified, sets a startdate. startdate starts at 00:00:00 + +-e: if specified, sets a enddate. enddate starts at 00:00:00 so if you wish to include enddate must add one more day. + +user: freeside username + +sftpuser: sftp user for arkftp.aninetworks.com + +sftppassword: password for sftp user + +=head1 EXAMPLES + +freeside-cdr-aninetworks-import -a <freeside user> <sftp login> <sftp password> +will get all cdr files starting from the day of the last day processed. + +freeside-cdr-aninetworks-import -s 20180120 -e 20180121 <freeside user> <sftp login> <sftp password> +will get all cdr files from 01/20/2018 + +freeside-cdr-aninetworks-import -v -s $(date --date="-1 day" +\%Y\%m\%d) -e $(date +\%Y\%m\%d) <freeside user> <sftp login> <sftp password> +will get all cdr files from yesterday + +=head1 BUGS + +=head1 SEE ALSO + +L<FS::cdr> + +=cut + +1;
\ No newline at end of file diff --git a/FS/bin/freeside-cdr-freeswitch b/FS/bin/freeside-cdr-freeswitch index 7f09578..3c18ef2 100644 --- a/FS/bin/freeside-cdr-freeswitch +++ b/FS/bin/freeside-cdr-freeswitch @@ -5,11 +5,12 @@ use Date::Parse 'str2time'; use FS::cdr::Import; FS::cdr::Import->dbi_import( - 'dbd' => 'Pg', - 'database' => 'fusionpbx', - 'table' => 'v_xml_cdr', - 'primary_key' => 'uuid', - 'column_map' => { #freeside => fusionpbx + 'dbd' => 'Pg', + 'database' => 'fusionpbx', + 'table' => 'v_xml_cdr', + 'status_column' => 'freesidestatus', + 'primary_key' => 'uuid', + 'column_map' => { #freeside => fusionpbx #'cdrid' => 'uuid', #Primary key #'' => 'CALL_SESSION_ID', # Call Session Id (unique per call session – GUID) 'uniqueid' => 'uuid', # |