X-Git-Url: http://git.freeside.biz/gitweb/?p=freeside.git;a=blobdiff_plain;f=FS%2Fbin%2Ffreeside-cdr-portaone-import;h=dfd130bbb1c518011d1175eb59bf6d33ebdf49c5;hp=be2555b10296d7589bd70f9d8d75581c26cffec9;hb=f822e27a1e00594332ffa487a1c284234c5580a6;hpb=0c66f548fe3c1a2e880c6672ff28a5c2da353057 diff --git a/FS/bin/freeside-cdr-portaone-import b/FS/bin/freeside-cdr-portaone-import index be2555b10..dfd130bbb 100644 --- a/FS/bin/freeside-cdr-portaone-import +++ b/FS/bin/freeside-cdr-portaone-import @@ -5,20 +5,25 @@ use strict; use Date::Format 'time2str'; use Date::Parse 'str2time'; use Getopt::Long; -use JSON; +use Cpanel::JSON::XS; use Net::HTTPS::Any qw(https_post); +use Time::Local; use FS::Record qw(qsearchs dbh); use FS::UID qw(adminsuidsetup); +use FS::cdr; +use FS::cdr_batch; sub usage { "Usage: +freeside-cdr-portaone-import -x exportnum [-s startdate] [-e enddate] [-v] freesideuser + freeside-cdr-portaone-import -h 'your.domain.com:443' -u switchusername -p switchpass [-s startdate] [-e enddate] [-v] freesideuser "; } -my ($host,$username,$password,$startdate,$enddate,$verbose); +my ($host,$username,$password,$startdate,$enddate,$verbose,$exportnum); GetOptions( "enddate=s" => \$enddate, "host=s" => \$host, @@ -26,11 +31,12 @@ GetOptions( "startdate=s" => \$startdate, "username=s" => \$username, "verbose" => \$verbose, + "x=s" => \$exportnum, ); my $fsuser = $ARGV[-1]; -die usage() unless $host && $password && $username && $fsuser; +die usage() unless $fsuser; adminsuidsetup($fsuser); @@ -40,6 +46,19 @@ if ($host =~ /^(.*)\:(.*)$/) { $port = $2; } +if ($exportnum) { + my $export = qsearchs('part_export', { 'exportnum' => $exportnum }); + die "Export not found" unless $export; + $host = $export->machine; + $port = $export->option('port'); + $username = $export->option('username'); + $password = $export->option('password'); + die "Auth info not fully specified in export" + unless $host && $port && $username && $password; +} + +die usage() unless $host && $password && $username; + if ($startdate) { $startdate = str2time($startdate) or die "Can't parse startdate $startdate"; $startdate = time2str("%Y-%m-%d %H:%M:%S",$startdate); @@ -50,11 +69,12 @@ unless ($startdate) { 'hashref' => { 'cdrbatch' => {op=>'like', value=>'portaone-import%'}}, 'order_by' => 'ORDER BY _date DESC LIMIT 1', }); - $startdate = time2str("%Y-%m-%d %H:%M:%S", $lastbatch->_date + 1) if $lastbatch; + $startdate = time2str("%Y-%m-%d %H:%M:%S", $lastbatch->_date) if $lastbatch; } $startdate ||= '2010-01-01 00:00:00'; #seems decently in the past -my $now = time; +my @now = localtime(); +my $now = timelocal(0,0,0,$now[3],$now[4],$now[5]); #most recent midnight if ($enddate) { $enddate = str2time($enddate) or die "Can't parse enddate $enddate"; $now = $enddate; @@ -62,6 +82,22 @@ if ($enddate) { } $enddate ||= time2str("%Y-%m-%d %H:%M:%S",$now); + +$FS::UID::AutoCommit = 0; + +my $cdrbatchname = 'portaone-import-'. time2str('%Y/%m/%d-%T',$now); +die "Batch $cdrbatchname already exists, please specify a different end date. " . usage() + if FS::cdr_batch->row_exists('cdrbatch = ?', $cdrbatchname); +my $cdr_batch = new FS::cdr_batch({ + 'cdrbatch' => $cdrbatchname, + '_date' => $now, +}); +my $error = $cdr_batch->insert; +if ($error) { + dbh->rollback; + die "Error creating batch: $error"; +} + print "Downloading records from $startdate to $enddate\n" if $verbose; my $auth_info; # needs to be declared undef for call_api @@ -74,58 +110,67 @@ my $results = {}; my $custlist = call_api('Customer','get_customer_list'); my @custnum = map { $_->{'i_customer'} } @{$custlist->{'customer_list'}}; foreach my $custnum (@custnum) { - print "Processing customer $custnum\n" if $verbose; - my $xdrs = call_api('Customer','get_customer_xdrs',{ - 'i_customer' => $custnum, - 'from_date' => $startdate, - 'to_date' => $enddate, - 'cdr_entity' => 'C', # can also be 'A', or blank for both - }); - my @xdrs = @{$xdrs->{'xdr_list'}}; - print scalar(@xdrs) . " xdrs retrieved\n" if $verbose; - $results->{$custnum} = \@xdrs; -} + print "Retrieving for customer $custnum\n" if $verbose; + my $step = 500; # too many records was crashing server, so we request in chunks + my $lastcount = $step; # to get the while loop rolling + my $totalcount = 0; # for verbose display only + my $offset = 0; + while ($lastcount == $step) { + my $xdrs = call_api('Customer','get_customer_xdrs',{ + 'i_customer' => $custnum, + 'from_date' => $startdate, + 'to_date' => $enddate, + 'cdr_entity' => 'A', + 'limit' => $step, + 'offset' => $offset, + }); + my @xdrs = @{$xdrs->{'xdr_list'}}; + print "Retrieved ".@xdrs." records\n" if $verbose && @xdrs; + my $skipped = 0; # for verbose display only + foreach my $xdr (@xdrs) { + if ( FS::cdr->row_exists('uniqueid = ?', $xdr->{'i_xdr'}) ) { + $skipped += 1; + next; + } + my $desc = $xdr->{'country'}; + if ($xdr->{'subdivision'}) { + $desc = ', ' . $desc if $desc; + $desc = $xdr->{'subdivision'} . $desc; + } + if ($xdr->{'description'}) { + $desc = ' (' . $desc . ')' if $desc; + $desc = $xdr->{'description'} . $desc; + } + my $cdr = FS::cdr->new ({ + 'cdrbatchnum' => $cdr_batch->cdrbatchnum, + 'uniqueid' => $xdr->{'i_xdr'}, + 'src' => $xdr->{'CLI'}, + 'dst' => $xdr->{'CLD'}, + 'upstream_price' => $xdr->{'charged_amount'}, + 'startdate' => $xdr->{'unix_connect_time'}, + 'enddate' => $xdr->{'unix_disconnect_time'}, + 'accountcode' => $xdr->{'account_id'}, + 'billsec' => $xdr->{'charged_quantity'}, + 'upstream_dst_regionname' => $desc, + }); + $error = $cdr->insert; + if ($error) { + dbh->rollback; + die "Error inserting cdr: $error"; + } + } #foreach $xdr + print "Skipped $skipped duplicate records\n" if $verbose && $skipped; + $totalcount += @xdrs - $skipped; + $lastcount = @xdrs; + $offset += $step; + } #while $lastcount == $step + print scalar($totalcount) . " records inserted\n" if $verbose; +} #foreach $custnum call_api('Session','logout',$auth_info); -$FS::UID::AutoCommit = 0; - -my $cdr_batch = new FS::cdr_batch({ - 'cdrbatch' => 'portaone-import-'. time2str('%Y/%m/%d-%T',$now), - '_date' => $now, -}); -my $error = $cdr_batch->insert; -if ($error) { - dbh->rollback; - die "Error inserting batch: $error"; -} - -foreach my $custnum (keys %$results) { - foreach my $xdr (@{$results->{$custnum}}) { - my $cdr = FS::cdr->new ({ - 'cdrbatchnum' => $cdr_batch->cdrbatchnum, - 'acctid' => $xdr->{'i_xdr'}, #??? - 'accountcode' => $xdr->{'account_id'}, #??? - 'startdate' => $xdr->{'unix_connect_time'}, - 'enddate' => $xdr->{'unix_disconnect_time'}, - -### XXX NEED TO MAP FIELDS: +### Full list of fields returned by API: #i_xdr int The unique ID of the xdr record -#account_id int The unique ID of the account database record -#CLI string Calling Line Identification -#CLD string Called Line Identification -#charged_amount float Amount charged -#charged_quantity int Units charged -#country string Country -#subdivision string Country subdivision -#description string Destination description -#disconnect_cause string The code of disconnect cause -#bill_status string Call bill status -#disconnect_reason string Call disconnect reason -#connect_time dateTime Call connect time -#unix_connect_time int Call connect time (expressed in Unix time format - seconds since epoch) -#disconnect_time dateTime Call disconnect time -#unix_disconnect_time int Call disconnect time i_xdr int The unique ID of the xdr record #account_id int The unique ID of the account database record #CLI string Calling Line Identification #CLD string Called Line Identification @@ -144,21 +189,8 @@ foreach my $custnum (keys %$results) { #bill_time dateTime Call bill time #bit_flags int Extended information how the service was used; the integer field that should be treated as a bit-map. Each currently used bit is listed in the Transaction_Flag_Types table (bit_offset indicates position) #call_recording_url string Path to recorded .wav files -#call_recording_server_url string URL to the recording server (expressed in Unix time format - seconds since epoch) -#bill_time dateTime Call bill time -#bit_flags int Extended information how the service was used; the integer field that should be treated as a bit-map. Each currently used bit is listed in the Transaction_Flag_Types table (bit_offset indicates position) -#call_recording_url string Path to recorded .wav files #call_recording_server_url string URL to the recording server - }); - $error = $cdr->insert; - if ($error) { - dbh->rollback; - die "Error inserting cdr: $error"; - } - } #foreach $xdr -} #foreach $custnum - dbh->commit; exit; @@ -167,7 +199,7 @@ sub call_api { my ($service,$method,$params) = @_; my %auth_info = $auth_info ? ('auth_info' => encode_json($auth_info)) : (); $params ||= {}; - print "Calling $service/$method...\n" if $verbose; + print "Calling $service/$method\n" if $verbose; my ( $page, $response, %reply_headers ) = https_post( 'host' => $host, 'port' => $port, @@ -175,6 +207,7 @@ sub call_api { 'args' => [ %auth_info, 'params' => encode_json($params) ], ); return decode_json($page) if $response eq '200 OK'; + dbh->rollback; if ($response =~ /^500/) { my $error = decode_json($page); die "Server returned error during $service/$method: ".$error->{'faultstring'}