use Date::Format 'time2str';
use Date::Parse 'str2time';
use Getopt::Long;
-use JSON;
+use Cpanel::JSON::XS;
use Net::HTTPS::Any qw(https_post);
+use Time::Local;
use FS::Record qw(qsearchs dbh);
use FS::UID qw(adminsuidsetup);
+use FS::cdr;
+use FS::cdr_batch;
sub usage {
"Usage:
+freeside-cdr-portaone-import -x exportnum [-s startdate] [-e enddate] [-v] freesideuser
+
freeside-cdr-portaone-import -h 'your.domain.com:443' -u switchusername -p switchpass
[-s startdate] [-e enddate] [-v] freesideuser
";
}
-my ($host,$username,$password,$startdate,$enddate,$verbose);
+my ($host,$username,$password,$startdate,$enddate,$verbose,$exportnum);
GetOptions(
"enddate=s" => \$enddate,
"host=s" => \$host,
"startdate=s" => \$startdate,
"username=s" => \$username,
"verbose" => \$verbose,
+ "x=s" => \$exportnum,
);
my $fsuser = $ARGV[-1];
-die usage() unless $host && $password && $username && $fsuser;
+die usage() unless $fsuser;
adminsuidsetup($fsuser);
$port = $2;
}
+if ($exportnum) {
+ my $export = qsearchs('part_export', { 'exportnum' => $exportnum });
+ die "Export not found" unless $export;
+ $host = $export->machine;
+ $port = $export->option('port');
+ $username = $export->option('username');
+ $password = $export->option('password');
+ die "Auth info not fully specified in export"
+ unless $host && $port && $username && $password;
+}
+
+die usage() unless $host && $password && $username;
+
if ($startdate) {
$startdate = str2time($startdate) or die "Can't parse startdate $startdate";
$startdate = time2str("%Y-%m-%d %H:%M:%S",$startdate);
'hashref' => { 'cdrbatch' => {op=>'like', value=>'portaone-import%'}},
'order_by' => 'ORDER BY _date DESC LIMIT 1',
});
- $startdate = time2str("%Y-%m-%d %H:%M:%S", $lastbatch->_date + 1) if $lastbatch;
+ $startdate = time2str("%Y-%m-%d %H:%M:%S", $lastbatch->_date) if $lastbatch;
}
$startdate ||= '2010-01-01 00:00:00'; #seems decently in the past
-my $now = time;
+my @now = localtime();
+my $now = timelocal(0,0,0,$now[3],$now[4],$now[5]); #most recent midnight
if ($enddate) {
$enddate = str2time($enddate) or die "Can't parse enddate $enddate";
$now = $enddate;
}
$enddate ||= time2str("%Y-%m-%d %H:%M:%S",$now);
+
+$FS::UID::AutoCommit = 0;
+
+my $cdrbatchname = 'portaone-import-'. time2str('%Y/%m/%d-%T',$now);
+die "Batch $cdrbatchname already exists, please specify a different end date. " . usage()
+ if FS::cdr_batch->row_exists('cdrbatch = ?', $cdrbatchname);
+my $cdr_batch = new FS::cdr_batch({
+ 'cdrbatch' => $cdrbatchname,
+ '_date' => $now,
+});
+my $error = $cdr_batch->insert;
+if ($error) {
+ dbh->rollback;
+ die "Error creating batch: $error";
+}
+
print "Downloading records from $startdate to $enddate\n" if $verbose;
my $auth_info; # needs to be declared undef for call_api
my $custlist = call_api('Customer','get_customer_list');
my @custnum = map { $_->{'i_customer'} } @{$custlist->{'customer_list'}};
foreach my $custnum (@custnum) {
- print "Processing customer $custnum\n" if $verbose;
- my $xdrs = call_api('Customer','get_customer_xdrs',{
- 'i_customer' => $custnum,
- 'from_date' => $startdate,
- 'to_date' => $enddate,
- 'cdr_entity' => 'C', # can also be 'A', or blank for both
- });
- my @xdrs = @{$xdrs->{'xdr_list'}};
- print scalar(@xdrs) . " xdrs retrieved\n" if $verbose;
- $results->{$custnum} = \@xdrs;
-}
+ print "Retrieving for customer $custnum\n" if $verbose;
+ my $step = 500; # too many records was crashing server, so we request in chunks
+ my $lastcount = $step; # to get the while loop rolling
+ my $totalcount = 0; # for verbose display only
+ my $offset = 0;
+ while ($lastcount == $step) {
+ my $xdrs = call_api('Customer','get_customer_xdrs',{
+ 'i_customer' => $custnum,
+ 'from_date' => $startdate,
+ 'to_date' => $enddate,
+ 'cdr_entity' => 'A',
+ 'limit' => $step,
+ 'offset' => $offset,
+ });
+ my @xdrs = @{$xdrs->{'xdr_list'}};
+ print "Retrieved ".@xdrs." records\n" if $verbose && @xdrs;
+ my $skipped = 0; # for verbose display only
+ foreach my $xdr (@xdrs) {
+ if ( FS::cdr->row_exists('uniqueid = ?', $xdr->{'i_xdr'}) ) {
+ $skipped += 1;
+ next;
+ }
+ my $desc = $xdr->{'country'};
+ if ($xdr->{'subdivision'}) {
+ $desc = ', ' . $desc if $desc;
+ $desc = $xdr->{'subdivision'} . $desc;
+ }
+ if ($xdr->{'description'}) {
+ $desc = ' (' . $desc . ')' if $desc;
+ $desc = $xdr->{'description'} . $desc;
+ }
+ my $cdr = FS::cdr->new ({
+ 'cdrbatchnum' => $cdr_batch->cdrbatchnum,
+ 'uniqueid' => $xdr->{'i_xdr'},
+ 'src' => $xdr->{'CLI'},
+ 'dst' => $xdr->{'CLD'},
+ 'upstream_price' => $xdr->{'charged_amount'},
+ 'startdate' => $xdr->{'unix_connect_time'},
+ 'enddate' => $xdr->{'unix_disconnect_time'},
+ 'accountcode' => $xdr->{'account_id'},
+ 'billsec' => $xdr->{'charged_quantity'},
+ 'upstream_dst_regionname' => $desc,
+ });
+ $error = $cdr->insert;
+ if ($error) {
+ dbh->rollback;
+ die "Error inserting cdr: $error";
+ }
+ } #foreach $xdr
+ print "Skipped $skipped duplicate records\n" if $verbose && $skipped;
+ $totalcount += @xdrs - $skipped;
+ $lastcount = @xdrs;
+ $offset += $step;
+ } #while $lastcount == $step
+ print scalar($totalcount) . " records inserted\n" if $verbose;
+} #foreach $custnum
call_api('Session','logout',$auth_info);
-$FS::UID::AutoCommit = 0;
-
-my $cdr_batch = new FS::cdr_batch({
- 'cdrbatch' => 'portaone-import-'. time2str('%Y/%m/%d-%T',$now),
- '_date' => $now,
-});
-my $error = $cdr_batch->insert;
-if ($error) {
- dbh->rollback;
- die "Error inserting batch: $error";
-}
-
-foreach my $custnum (keys %$results) {
- foreach my $xdr (@{$results->{$custnum}}) {
- my $cdr = FS::cdr->new ({
- 'cdrbatchnum' => $cdr_batch->cdrbatchnum,
- 'acctid' => $xdr->{'i_xdr'}, #???
- 'accountcode' => $xdr->{'account_id'}, #???
- 'startdate' => $xdr->{'unix_connect_time'},
- 'enddate' => $xdr->{'unix_disconnect_time'},
-
-### XXX NEED TO MAP FIELDS:
+### Full list of fields returned by API:
#i_xdr int The unique ID of the xdr record
-#account_id int The unique ID of the account database record
-#CLI string Calling Line Identification
-#CLD string Called Line Identification
-#charged_amount float Amount charged
-#charged_quantity int Units charged
-#country string Country
-#subdivision string Country subdivision
-#description string Destination description
-#disconnect_cause string The code of disconnect cause
-#bill_status string Call bill status
-#disconnect_reason string Call disconnect reason
-#connect_time dateTime Call connect time
-#unix_connect_time int Call connect time (expressed in Unix time format - seconds since epoch)
-#disconnect_time dateTime Call disconnect time
-#unix_disconnect_time int Call disconnect time i_xdr int The unique ID of the xdr record
#account_id int The unique ID of the account database record
#CLI string Calling Line Identification
#CLD string Called Line Identification
#bill_time dateTime Call bill time
#bit_flags int Extended information how the service was used; the integer field that should be treated as a bit-map. Each currently used bit is listed in the Transaction_Flag_Types table (bit_offset indicates position)
#call_recording_url string Path to recorded .wav files
-#call_recording_server_url string URL to the recording server (expressed in Unix time format - seconds since epoch)
-#bill_time dateTime Call bill time
-#bit_flags int Extended information how the service was used; the integer field that should be treated as a bit-map. Each currently used bit is listed in the Transaction_Flag_Types table (bit_offset indicates position)
-#call_recording_url string Path to recorded .wav files
#call_recording_server_url string URL to the recording server
- });
- $error = $cdr->insert;
- if ($error) {
- dbh->rollback;
- die "Error inserting cdr: $error";
- }
- } #foreach $xdr
-} #foreach $custnum
-
dbh->commit;
exit;
my ($service,$method,$params) = @_;
my %auth_info = $auth_info ? ('auth_info' => encode_json($auth_info)) : ();
$params ||= {};
- print "Calling $service/$method...\n" if $verbose;
+ print "Calling $service/$method\n" if $verbose;
my ( $page, $response, %reply_headers ) = https_post(
'host' => $host,
'port' => $port,
'args' => [ %auth_info, 'params' => encode_json($params) ],
);
return decode_json($page) if $response eq '200 OK';
+ dbh->rollback;
if ($response =~ /^500/) {
my $error = decode_json($page);
die "Server returned error during $service/$method: ".$error->{'faultstring'}
projects / freeside.git / blobdiff