diff options
Diffstat (limited to 'FS/bin')
| -rwxr-xr-x | FS/bin/freeside-cdr-a2billing-import | 12 | ||||
| -rwxr-xr-x | FS/bin/freeside-cdr-conexiant-import | 128 | ||||
| -rwxr-xr-x | FS/bin/freeside-cdr-evariste-import | 2 | ||||
| -rw-r--r-- | FS/bin/freeside-cdrrewrited | 93 | ||||
| -rw-r--r-- | FS/bin/freeside-ipifony-download | 14 | ||||
| -rwxr-xr-x | FS/bin/freeside-upgrade | 102 |
6 files changed, 278 insertions, 73 deletions
diff --git a/FS/bin/freeside-cdr-a2billing-import b/FS/bin/freeside-cdr-a2billing-import index a8469e744..aa7fa4a61 100755 --- a/FS/bin/freeside-cdr-a2billing-import +++ b/FS/bin/freeside-cdr-a2billing-import @@ -120,7 +120,13 @@ my $updates = 0; my $row; while ( $row = $sth->fetchrow_hashref ) { - $row->{calledstation} =~ s/^1//; + my $dst = $row->{calledstation}; + my $dst_ip_addr = ''; + if ($dst =~ m[^SIP/(\d+)@(.*)$] ) { + $dst = $1; + $dst_ip_addr = $2; + } + $dst =~ s/^1//; $row->{src} =~ s/^1//; my $cdr = FS::cdr->new ({ uniqueid => $row->{sessionid}, @@ -129,8 +135,10 @@ while ( $row = $sth->fetchrow_hashref ) { enddate => time2str($row->{stoptime}), duration => $row->{sessiontime}, billsec => $row->{real_sessiontime}, - dst => $row->{calledstation}, + dst => $dst, src => $row->{src}, + dst_ip_addr => $dst_ip_addr, + dstchannel => $row->{calledstation}, charged_party => $row->{username}, upstream_rateplanid => $row->{id_tariffplan}, upstream_rateid => $row->{id_ratecard}, # I think? diff --git a/FS/bin/freeside-cdr-conexiant-import b/FS/bin/freeside-cdr-conexiant-import new file mode 100755 index 000000000..f2b469111 --- /dev/null +++ b/FS/bin/freeside-cdr-conexiant-import @@ -0,0 +1,128 @@ +#!/usr/bin/perl + +use strict; + +use Cpanel::JSON::XS; +use Getopt::Long; +use LWP::UserAgent; +use MIME::Base64; +use Net::HTTPS::Any qw(https_post https_get); + +use FS::UID qw(adminsuidsetup); +use FS::Record qw(qsearchs); +use FS::cdr; +use FS::cdr_batch; + +sub usage { +"Usage: +freeside-cdr-conexiant-import -h -u username -p apikey [-v] freesideuser + +Downloads any existing CDR files with the BilledCallsOnly flag and +imports records that have not been imported yet. Silently skips +records that have already been imported. +"; +} + +# should really be using a module for this +`which unzip` or die "can't find unzip executable"; + +my ($username,$password,$verbose); +GetOptions( + "password=s" => \$password, + "username=s" => \$username, + "verbose" => \$verbose, +); + +my $fsuser = $ARGV[-1]; + +die usage() unless $fsuser; + +adminsuidsetup($fsuser); + +my ( $page, $response, %reply_headers ) = https_post( + 'host' => 'api.conexiant.net', + 'port' => '443', + 'path' => '/v1/Cdrs/SearchCdrsDownloads', + 'headers' => { + 'Authorization' => 'Basic ' . MIME::Base64::encode("$username:$password",'') + }, + 'content' => '{}', +); + +die "Bad response from conexiant server: $response" + unless $response =~ /^200/; + +my $result = decode_json($page); + +die "Error from conexiant: " . ($result->{'ErrorInfo'} || 'No error message') + unless $result->{'Success'}; + +my $files = $result->{'Data'}->{'Result'}; + +die "Unexpected results from conexiant, not an array" + unless ref($files) eq 'ARRAY'; + +my $dir = $FS::UID::cache_dir. "/cache.". $FS::UID::datasrc; +my $ua = LWP::UserAgent->new; + +# Download files are created automatically at regular frequent intervals, +# but they contain overlapping data. +# +# FS::cdr::conexiant automatically skips previously imported cdrs +foreach my $file (@$files) { + next unless $file->{'BilledCallsOnly'}; + my $cdrbatch = 'conexiant-' . $file->{'Identifier'}; + # files that contained no new records will unfortunately be re-downloaded, + # but the alternative is to leave an excess of empty batches in system, + # and re-downloading is harmless (all files expire after 48 hours anyway) + if (qsearchs('cdr_batch',{ 'cdrbatch' => $cdrbatch })) { + print "$cdrbatch already imported\n" if $verbose; + next; + } + if ($verbose) { + print "Downloading $cdrbatch\n". + " Created ".$file->{'CreatedOn'}."\n". + " Start ".$file->{'QueryStart'}."\n". + " End ".$file->{'QueryEnd'}."\n". + " Link ".$file->{'ValidLink'}."\n"; + } + my $zfh = new File::Temp( TEMPLATE => 'conexiant.XXXXXXXX', + SUFFIX => '.zip', + DIR => $dir, + ) + or die "can't open temporary file to store download: $!\n"; + my $cfh = new File::Temp( TEMPLATE => 'conexiant.XXXXXXXX', + SUFFIX => '.csv', + DIR => $dir, + ) + or die "can't open temporary file to unzip download: $!\n"; + # yeah, these files ain't secured in any way + my $response = $ua->get($file->{'ValidLink'}, ':content_file' => $zfh->filename); + unless ($response->is_success) { + die "Error downloading $cdrbatch: ".$response->status_line; + } + my $zfilename = $zfh->filename; + print $cfh `unzip -p $zfilename 'Conexiant Cdrs.csv'`; + seek($cfh,0,0); + print "Importing batch $cdrbatch\n" if $verbose; + my $error = FS::cdr::batch_import({ + 'batch_namevalue' => $cdrbatch, + 'file' => $cfh->filename, + 'format' => 'conexiant' + }); + if ($error eq 'Empty file!') { + print "File contains no records\n" if $verbose; + $error = ''; + } elsif ($error eq "All records in file were previously imported") { + print "File contains no new cdrs, no batch created\n" if $verbose; + $error = ''; + } elsif ($verbose && !$error) { + print "File successfully imported\n"; + } + die "Error importing $cdrbatch: $error" if $error; +} + +exit; + + + diff --git a/FS/bin/freeside-cdr-evariste-import b/FS/bin/freeside-cdr-evariste-import index 0487ae539..d5e13f98c 100755 --- a/FS/bin/freeside-cdr-evariste-import +++ b/FS/bin/freeside-cdr-evariste-import @@ -100,7 +100,7 @@ while (my $row = $csth->fetchrow_hashref) { 'cdrbatchnum' => $cdr_batch->cdrbatchnum, 'uniqueid' => $row->{'id'}, 'src' => $row->{'src'}, - 'dst' => $row->{'dest'}, + 'dst' => $row->{'routing_target'} || $row->{'dest'}, # dest_orig? dest_trans? 'startdate' => int(str2time($row->{'start_time'})), 'answerdate' => int(str2time($row->{'answer_time'})), 'enddate' => int(str2time($row->{'end_time'})), diff --git a/FS/bin/freeside-cdrrewrited b/FS/bin/freeside-cdrrewrited index 16f931fbf..34a206849 100644 --- a/FS/bin/freeside-cdrrewrited +++ b/FS/bin/freeside-cdrrewrited @@ -4,7 +4,7 @@ use strict; use vars qw( $conf ); use FS::Daemon ':all'; #daemonize1 drop_root daemonize2 myexit logfile sig* use FS::UID qw( adminsuidsetup ); -use FS::Record qw( qsearch qsearchs ); +use FS::Record qw( qsearch qsearchs dbh ); #use FS::cdr; #use FS::cust_pkg; #use FS::queue; @@ -24,12 +24,12 @@ daemonize2(); $conf = new FS::Conf; -die "not running; cdr-asterisk_forward_rewrite, cdr-charged_party_rewrite ". - " and cdr-taqua-accountcode_rewrite conf options are all off\n" +die "not running; relevant conf options are all off\n" unless _shouldrun(); #-- +#used for taqua my %sessionnum_unmatch = (); my $sessionnum_retry = 4 * 60 * 60; # 4 hours my $sessionnum_giveup = 4 * 24 * 60 * 60; # 4 days @@ -45,20 +45,25 @@ while (1) { # instead of just doing this search like normal CDRs #hmm :/ + #used only by taqua, should have no effect otherwise my @recent = grep { ($sessionnum_unmatch{$_} + $sessionnum_retry) > time } keys %sessionnum_unmatch; my $extra_sql = scalar(@recent) ? ' AND acctid NOT IN ('. join(',', @recent). ') ' : ''; + #order matters for removing dupes--only the first is preserved + $extra_sql .= ' ORDER BY acctid ' + if $conf->exists('cdr-skip_duplicate_rewrite'); + my $found = 0; - my %skip = (); + my %skip = (); #used only by taqua my %warning = (); foreach my $cdr ( qsearch( { 'table' => 'cdr', - 'extra_sql' => 'FOR UPDATE', + 'extra_sql' => 'FOR UPDATE', #XXX overwritten by opt below...would fixing this break anything? 'hashref' => {}, 'extra_sql' => 'WHERE freesidestatus IS NULL '. ' AND freesiderewritestatus IS NULL '. @@ -67,11 +72,27 @@ while (1) { } ) ) { - next if $skip{$cdr->acctid}; + next if $skip{$cdr->acctid}; #used only by taqua $found = 1; my @status = (); + if ($conf->exists('cdr-skip_duplicate_rewrite')) { + #qsearch can't handle timestamp type of calldate + my $sth = dbh->prepare( + 'SELECT 1 FROM cdr WHERE src=? AND dst=? AND calldate=? AND acctid < ? LIMIT 1' + ) or die dbh->errstr; + $sth->execute($cdr->src,$cdr->dst,$cdr->calldate,$cdr->acctid) or die $sth->errstr; + my $isdup = $sth->fetchrow_hashref; + $sth->finish; + if ($isdup) { + #we only act on this cdr, not touching previous dupes + #if a dupe somehow creeped in previously, too late to fix it + $cdr->freesidestatus('done'); #prevent it from being billed + push(@status,'duplicate'); + } + } + if ( $conf->exists('cdr-asterisk_forward_rewrite') && $cdr->dstchannel =~ /^Local\/(\d+)/i && $1 ne $cdr->dst ) @@ -198,6 +219,18 @@ while (1) { } + if ( $conf->exists('cdr-userfield_dnis_rewrite') and + $cdr->userfield =~ /DNIS=(\d+)/ ) { + $cdr->dst($1); + push @status, 'userfield_dnis'; + } + + if ( $conf->exists('cdr-intl_to_domestic_rewrite') and + $cdr->dst =~ /^(011)(\d{0,7})$/ ) { + $cdr->dst($2); + push @status, 'intl_to_domestic'; + } + $cdr->freesiderewritestatus( scalar(@status) ? join('/', @status) : 'skipped' ); @@ -233,6 +266,9 @@ sub _shouldrun { || $conf->exists('cdr-charged_party_rewrite') || $conf->exists('cdr-taqua-accountcode_rewrite') || $conf->exists('cdr-taqua-callerid_rewrite') + || $conf->exists('cdr-intl_to_domestic_rewrite') + || $conf->exists('cdr-userfield_dnis_rewrite') + || $conf->exists('cdr-skip_duplicate_rewrite') || 0 ; } @@ -252,8 +288,49 @@ freeside-cdrrewrited - Real-time daemon for CDR rewriting =head1 DESCRIPTION Runs continuously, searches for CDRs and does forwarded-call rewriting if any -of the "cdr-asterisk_forward_rewrite", "cdr-charged_party_rewrite" or -"cdr-taqua-accountcode_rewrite" config options are enabled. +of the following config options are enabled: + +=over 4 + +=item cdr-skip_duplicate_rewrite + +Marks as 'done' (prevents billing for) any CDRs with +a src, dst and calldate identical to an existing CDR + +=item cdr-asterisk_australia_rewrite + +Classifies Australian numbers as domestic, mobile, tollfree, international, or +"other", and tries to assign a cdrtypenum based on that. + +=item cdr-asterisk_forward_rewrite + +Identifies Asterisk forwarded calls using the 'dstchannel' field. If the +dstchannel is "Local/" followed by a number, but the number doesn't match the +dst field, the dst field will be rewritten to match. + +=item cdr-charged_party_rewrite + +Calls set_charged_party on all calls. + +=item cdr-taqua-accountcode_rewrite + +=item cdr-taqua-callerid_rewrite + +These actually have the same effect. Taqua uses cdrtypenum = 1 to tag accessory +records. They will have "sessionnum" = that of the primary record, and +"lastapp" indicating their function: + +- "acctcode": "lastdata" contains the dialed account code. Insert this into the +accountcode field of the primary record. + +- "CallerId": "lastdata" contains "allowed" or "restricted". If "restricted" +then the clid field of the primary record is set to "PRIVATE". + +=item cdr-intl_to_domestic_rewrite + +Finds records where the destination number has the "011" international prefix, +but with seven or fewer digits in the rest of the number, and strips the "011" +prefix so that they will be treated as domestic calls. This is very uncommon. =head1 SEE ALSO diff --git a/FS/bin/freeside-ipifony-download b/FS/bin/freeside-ipifony-download index ee1f4bdfe..10faa7483 100644 --- a/FS/bin/freeside-ipifony-download +++ b/FS/bin/freeside-ipifony-download @@ -13,7 +13,7 @@ use File::Copy qw(copy); use Text::CSV; my %opt; -getopts('vqa:P:C:e:', \%opt); +getopts('vqNa:P:C:e:', \%opt); # Product codes that are subject to flat rate E911 charges. For these # products, the'quantity' field represents the number of lines. @@ -32,6 +32,7 @@ sub HELP_MESSAGE { ' freeside-ipifony-download [ -v ] [ -q ] + [ -N ] [ -a archivedir ] [ -P port ] [ -C category ] @@ -192,7 +193,8 @@ FILE: foreach my $filename (@$files) { if ( $next_bill_date ) { my ($bill_month, $bill_year) = (localtime($next_bill_date))[4, 5]; my ($this_month, $this_year) = (localtime(time))[4, 5]; - if ( $this_month == $bill_month and $this_year == $bill_year ) { + if ( $opt{N} or + $this_month == $bill_month and $this_year == $bill_year ) { $cust_main->set('charge_date', $next_bill_date); } } @@ -296,6 +298,7 @@ freeside-ipifony-download - Download and import invoice items from IPifony. freeside-ipifony-download [ -v ] [ -q ] + [ -N ] [ -a archivedir ] [ -P port ] [ -C category ] @@ -312,12 +315,19 @@ have an authorization key to connect as that user. I<hostname>: the SFTP server. +I<path>: the path on the server to the working directory. The working +directory is the one containing the "ready/" and "done/" subdirectories. + =head1 OPTIONAL PARAMETERS -v: Be verbose. -q: Include the quantity and unit price in the charge description. +-N: Always bill the charges on the customer's next bill date, if they have +one. Otherwise, charges will be billed on the next bill date only if it's +within the current calendar month. + -a I<archivedir>: Save a copy of the downloaded file to I<archivedir>. -P I<port>: Connect to that TCP port. diff --git a/FS/bin/freeside-upgrade b/FS/bin/freeside-upgrade index 9c2811538..77087c373 100755 --- a/FS/bin/freeside-upgrade +++ b/FS/bin/freeside-upgrade @@ -61,76 +61,37 @@ $start = time; my @bugfix = (); -if (dbdef->table('cust_main')->column('agent_custid') && ! $opt_s) { - push @bugfix, - "UPDATE cust_main SET agent_custid = NULL where agent_custid = ''"; - - push @bugfix, - "UPDATE h_cust_main SET agent_custid = NULL where agent_custid = ''" - if (dbdef->table('h_cust_main')); -} - -if ( dbdef->table('cgp_rule_condition') && - dbdef->table('cgp_rule_condition')->column('condition') - ) -{ - push @bugfix, - "ALTER TABLE ${_}cgp_rule_condition RENAME COLUMN condition TO conditionname" - for '', 'h_'; - -} - -if ( dbdef->table('areacode') and - dbdef->table('areacode')->primary_key eq 'code' ) -{ - if ( driver_name =~ /^mysql/i ) { - push @bugfix, - 'ALTER TABLE areacode DROP PRIMARY KEY', - 'ALTER TABLE areacode ADD COLUMN (areanum int auto_increment primary key)'; - } - else { - push @bugfix, 'ALTER TABLE areacode DROP CONSTRAINT areacode_pkey'; - } -} - -if ( dbdef->table('upgrade_journal') ) { - if ( driver_name =~ /^Pg/i ) { - push @bugfix, " - SELECT SETVAL( 'upgrade_journal_upgradenum_seq', - ( SELECT MAX(upgradenum) FROM upgrade_journal ) - ) - "; - #MySQL can't do this in a statement so have to do it manually - #} elsif ( driver_name =~ /^mysql/i ) { - # push @bugfix, " - # ALTER TABLE upgrade_journal AUTO_INCREMENT = - # ( ( SELECT MAX(upgradenum) FROM upgrade_journal ) + 1 ) - # "; - } -} - if ( $DRY_RUN ) { - print - join(";\n", @bugfix ). ";\n"; -} elsif ( @bugfix ) { - + print join(";\n", @bugfix ). ";\n"; +} else { foreach my $statement ( @bugfix ) { warn "$statement\n"; $dbh->do( $statement ) or die "Error: ". $dbh->errstr. "\n executing: $statement"; } +} +### +# Fixes before schema upgrade +### +# this isn't actually the main schema upgrade, this calls _upgrade_schema +# in any class that has it +if ( $DRY_RUN ) { + #XXX no dry run for upgrade_schema stuff yet. + # looking at the code some are a mix of SQL statements and our methods, icky. + # its not like dry run is 100% anyway, all sort of other later upgrade tasks + # aren't printed either +} else { upgrade_schema(%upgrade_opts); dbdef_create($dbh, $dbdef_file); delete $FS::Schema::dbdef_cache{$dbdef_file}; #force an actual reload reload_dbdef($dbdef_file); - } -#you should have run fs-migrate-part_svc ages ago, when you upgraded -#from 1.3 to 1.4... if not, it needs to be hooked into -upgrade here or -#you'll lose all the part_svc settings it migrates to part_svc_column +### +# Now here is the main/automatic schema upgrade via DBIx::DBSchema +### my $conf = new FS::Conf; @@ -144,10 +105,12 @@ my @statements = dbdef->sql_update_schema( $dbdef_dist, { 'nullify_default' => 1, }, ); -#### NEW CUSTOM FIELDS: +### +# New custom fields +### # 1. prevent new custom field columns from being dropped by upgrade # 2. migrate old virtual fields to real fields (new custom fields) -#### + my $cfsth = $dbh->prepare("SELECT * FROM part_virtual_field") or die $dbh->errstr; $cfsth->execute or die $cfsth->errstr; @@ -168,6 +131,10 @@ while ( $cf = $cfsth->fetchrow_hashref ) { } warn "Custom fields schema upgrade completed"; +### +# Other stuff +### + @statements = grep { $_ !~ /^CREATE +INDEX +h_queue/i } #useless, holds up queue insertion @statements; @@ -194,7 +161,10 @@ if ( $opt_c ) { } -my $MAX_HANDLES; # undef for now, set it if you want a limit + +### +# Now run the @statements +### if ( $DRY_RUN ) { print @@ -202,6 +172,12 @@ if ( $DRY_RUN ) { exit; } elsif ( $opt_a ) { + ### + # -a: Run schema changes in parallel (Pg only). + ### + + my $MAX_HANDLES; # undef for now, set it if you want a limit + my @phases = map { [] } 0..4; my $fsupgrade_idx = 1; my %idx_map; @@ -293,7 +269,13 @@ if ( $DRY_RUN ) { # $start = time; # dbdef->update_schema( dbdef_dist(datasrc), $dbh ); -} else { # normal case, run statements sequentially + +} else { + + ### + # normal case, run statements sequentially + ### + foreach my $statement ( @statements ) { warn "$statement\n"; $dbh->do( $statement ) |