diff options
author | Ivan Kohler <ivan@freeside.biz> | 2015-03-05 17:30:46 -0800 |
---|---|---|
committer | Ivan Kohler <ivan@freeside.biz> | 2015-03-05 17:30:46 -0800 |
commit | f34ac99a5e160d2eac4fb2cfecce820d1cf9b7ed (patch) | |
tree | efb6c09646d61677fab7ac1d42bf14117a331dff /FS | |
parent | aaa1a91c209ec3547b2c62dc59af935649a62b58 (diff) | |
parent | 0668c6afdce579dd202b12874bc0e9f1abe40ada (diff) |
Merge branch 'master' of git.freeside.biz:/home/git/freeside
Diffstat (limited to 'FS')
-rw-r--r-- | FS/FS/Conf.pm | 1 | ||||
-rw-r--r-- | FS/FS/Misc/Geo.pm | 51 | ||||
-rw-r--r-- | FS/FS/Report/Table/Daily.pm | 57 | ||||
-rw-r--r-- | FS/FS/access_user.pm | 113 | ||||
-rw-r--r-- | FS/FS/cust_pay.pm | 20 | ||||
-rw-r--r-- | FS/FS/cust_pay_batch.pm | 18 | ||||
-rwxr-xr-x | FS/bin/freeside-cdr-mysql | 88 | ||||
-rwxr-xr-x | FS/bin/freeside-cdr-thinktel | 127 | ||||
-rwxr-xr-x | FS/bin/freeside-cdr-voip_ms | 70 |
9 files changed, 498 insertions, 47 deletions
diff --git a/FS/FS/Conf.pm b/FS/FS/Conf.pm index 479e9ab2b..d5e8960b2 100644 --- a/FS/FS/Conf.pm +++ b/FS/FS/Conf.pm @@ -4440,6 +4440,7 @@ and customer address. Include units.', 'section' => 'invoicing', 'description' => 'Instead of showing payments (and credits) applied to the invoice, show those received since the previous invoice date.', 'type' => 'checkbox', + 'uscensus' => 'U.S. Census Bureau', }, { diff --git a/FS/FS/Misc/Geo.pm b/FS/FS/Misc/Geo.pm index dbc383a14..1e17b5fcc 100644 --- a/FS/FS/Misc/Geo.pm +++ b/FS/FS/Misc/Geo.pm @@ -236,6 +236,8 @@ sub wa_sales { die "WA tax district lookup error: $error"; } +###### USPS Standardization ###### + sub standardize_usps { my $class = shift; @@ -292,6 +294,55 @@ sub standardize_usps { addr_clean=> 'Y' } } +###### U.S. Census Bureau ###### + +sub standardize_uscensus { + my $self = shift; + my $location = shift; + + eval "use Geo::USCensus::Geocoding"; + die $@ if $@; + + if ( $location->{country} ne 'US' ) { + # soft failure + warn "standardize_uscensus not for use in country ".$location->{country}."\n"; + $location->{addr_clean} = ''; + return $location; + } + + my $request = { + street => $location->{address1}, + city => $location->{city}, + state => $location->{state}, + zip => $location->{zip}, + debug => ($DEBUG || 0), + }; + + my $result = Geo::USCensus::Geocoding->query($request); + if ( $result->is_match ) { + # unfortunately we get the address back as a single line + if ($result->address =~ /^(.*), (.*), ([A-Z]{2}), (\d{5}.*)$/) { + return +{ + address1 => $1, + city => $2, + state => $3, + zip => $4, + address2 => uc($location->{address2}), + latitude => $result->latitude, + longitude => $result->longitude, + censustract => $result->censustract, + }; + } else { + die "can't parse address '".$result->address."'"; + } + } else { + warn Dumper($result) if $DEBUG; + die $result->error_message; + } +} + +####### EZLOCATE (obsolete) ####### + sub _tomtom_query { # helper method for the below my %args = @_; my $result = Geo::TomTom::Geocoding->query(%args); diff --git a/FS/FS/Report/Table/Daily.pm b/FS/FS/Report/Table/Daily.pm index c18106461..66739379d 100644 --- a/FS/FS/Report/Table/Daily.pm +++ b/FS/FS/Report/Table/Daily.pm @@ -1,13 +1,9 @@ package FS::Report::Table::Daily; use strict; -use vars qw( @ISA ); -use FS::Report::Table; +use base 'FS::Report::Table'; +use DateTime; use FS::Conf; -use Time::Local qw( timelocal timelocal_nocheck ); # eventually replace with DateTime -use Date::Format qw( time2str ); - -@ISA = qw( FS::Report::Table ); =head1 NAME @@ -50,35 +46,51 @@ sub data { my $sday = $self->{'start_day'}; my $smonth = $self->{'start_month'}; - my $syear = $self->{'start_year'}; + my $syear = $self->{'start_year'} + 1900; # temporary kludge my $eday = $self->{'end_day'}; my $emonth = $self->{'end_month'}; - my $eyear = $self->{'end_year'}; + my $eyear = $self->{'end_year'} + 1900; my $agentnum = $self->{'agentnum'}; my $cust_classnum = $self->{'cust_classnum'} || []; $cust_classnum = [ $cust_classnum ] if !ref($cust_classnum); - my %data; + #these need to get generalized, sheesh + my %data = ( + # rows (time intervals) + speriod => [], # start timestamps + eperiod => [], # end timestamps + label => [], # date labels + data => [], # arrayrefs of column values + + # columns (observables + query parameters) + items => $self->{'items'}, + item_labels => $self->{'item_labels'} || $self->{'items'}, + colors => $self->{'colors'}, # no default? + links => $self->{'links'} || [], + ); - my $sdate = timelocal(0,0,0,$sday,$smonth-1,$syear); - my $edate = timelocal(0,0,0,$eday,$emonth-1,$eyear); + my $sdate = DateTime->new( + day => $sday, + month => $smonth, + year => $syear, + time_zone => 'local' + ); + my $edate = DateTime->new( + day => $eday, + month => $emonth, + year => $eyear, + time_zone => 'local' + )->add(days => 1); # include all of the end day my $conf = FS::Conf->new; my $date_format = $conf->config('date_format') || '%d/%m/%Y'; - #warn "daily range $sdate $edate\n"; - - # XXX: use date_format config for the labels since we have day in the labels now? while ( $sdate < $edate ) { - push @{$data{label}}, time2str($date_format, $sdate); + push @{$data{label}}, $sdate->strftime($date_format); - my $speriod = $sdate; - - #ala part_pkg->add_freq, to deal with local DST. DateTime also a good idea - my ($mday,$mon,$year) = (localtime($sdate) )[3,4,5]; - $sdate = timelocal_nocheck(0,0,0,$mday+1,$mon,$year); - - my $eperiod = $sdate; + my $speriod = $sdate->epoch; + $sdate->add(days => 1); + my $eperiod = $sdate->epoch;; push @{$data{speriod}}, $speriod; push @{$data{eperiod}}, $eperiod; @@ -95,7 +107,6 @@ sub data { } } - #these need to get generalized, sheesh $data{'items'} = $self->{'items'}; $data{'item_labels'} = $self->{'item_labels'} || $self->{'items'}; $data{'colors'} = $self->{'colors'}; diff --git a/FS/FS/access_user.pm b/FS/FS/access_user.pm index 44d3beec1..68d2deaba 100644 --- a/FS/FS/access_user.pm +++ b/FS/FS/access_user.pm @@ -354,7 +354,9 @@ sub agentnums_sql { if ( $self->access_right($viewall_right) ) { push @or, "$agentnum IS NOT NULL"; } else { - push @or, "$agentnum IN (". join(',', $self->agentnums). ')'; + my @agentnums = $self->agentnums; + push @or, "$agentnum IN (". join(',', @agentnums). ')' + if @agentnums; } push @or, "$agentnum IS NULL" @@ -370,17 +372,24 @@ sub agentnums_sql { Returns true if the user can view the specified agent. +Also accepts optional hashref cache, to avoid redundant database calls. + =cut sub agentnum { - my( $self, $agentnum ) = @_; + my( $self, $agentnum, $cache ) = @_; + $cache ||= {}; + return $cache->{$self->usernum}->{$agentnum} + if $cache->{$self->usernum}->{$agentnum}; my $sth = dbh->prepare( "SELECT COUNT(*) FROM access_usergroup JOIN access_groupagent USING ( groupnum ) WHERE usernum = ? AND agentnum = ?" ) or die dbh->errstr; $sth->execute($self->usernum, $agentnum) or die $sth->errstr; - $sth->fetchrow_arrayref->[0]; + $cache->{$self->usernum}->{$agentnum} = $sth->fetchrow_arrayref->[0]; + $sth->finish; + return $cache->{$self->usernum}->{$agentnum}; } =item agents [ HASHREF | OPTION => VALUE ... ] @@ -400,6 +409,104 @@ sub agents { }); } +=item access_users [ HASHREF | OPTION => VALUE ... ] + +Returns an array of FS::access_user objects, one for each non-disabled +access_user in the system that shares an agent (via group membership) with +the invoking object. Regardless of options and agents, will always at +least return the invoking user and any users who have viewall_right. + +Accepts the following options: + +=over 4 + +=item table + +Only return users who appear in the usernum field of this table + +=item disabled + +Include disabled users if true (defaults to false) + +=item viewall_right + +All users will be returned if the current user has the provided +access right, regardless of agents (other filters still apply.) +Defaults to 'View customers of all agents' + +=cut + +#Leaving undocumented until such time as this functionality is actually used +# +#=item null +# +#Users with no agents will be returned. +# +#=item null_right +# +#Users with no agents will be returned if the current user has the provided +#access right. + +sub access_users { + my $self = shift; + my %opt = ref($_[0]) ? %{$_[0]} : @_; + my $table = $opt{'table'}; + my $search = { 'table' => 'access_user' }; + $search->{'hashref'} = $opt{'disabled'} ? {} : { 'disabled' => '' }; + $search->{'addl_from'} = "INNER JOIN $table ON (access_user.usernum = $table.usernum)" + if $table; + my @access_users = qsearch($search); + my $viewall_right = $opt{'viewall_right'} || 'View customers of all agents'; + return @access_users if $self->access_right($viewall_right); + #filter for users with agents $self can view + my @out; + my $agentnum_cache = {}; +ACCESS_USER: + foreach my $access_user (@access_users) { + # you can always view yourself, regardless of agents, + # and you can always view someone who can view you, + # since they might have affected your customers + if ( ($self->usernum eq $access_user->usernum) + || $access_user->access_right($viewall_right) + ) { + push(@out,$access_user); + next; + } + # if user has no agents, you need null or null_right to view + my @agents = $access_user->agents('viewall_right'=>'NONE'); #handled viewall_right above + if (!@agents) { + if ( $opt{'null'} || + ( $opt{'null_right'} && $self->access_right($opt{'null_right'}) ) + ) { + push(@out,$access_user); + } + next; + } + # otherwise, you need an agent in common + foreach my $agent (@agents) { + if ($self->agentnum($agent->agentnum,$agentnum_cache)) { + push(@out,$access_user); + next ACCESS_USER; + } + } + } + return @out; +} + +=item access_users_hashref [ HASHREF | OPTION => VALUE ... ] + +Accepts same options as L</access_users>. Returns a hashref of +users, with keys of usernum and values of username. + +=cut + +sub access_users_hashref { + my $self = shift; + my %access_users = map { $_->usernum => $_->username } + $self->access_users(@_); + return \%access_users; +} + =item access_right RIGHTNAME | LISTREF Given a right name or a list reference of right names, returns true if this diff --git a/FS/FS/cust_pay.pm b/FS/FS/cust_pay.pm index ce8a3552f..8b4c98ad6 100644 --- a/FS/FS/cust_pay.pm +++ b/FS/FS/cust_pay.pm @@ -1170,8 +1170,6 @@ sub process_upgrade_paybatch { sub process_batch_import { my $job = shift; - #agent_custid isn't a cust_pay field, see hash callback - my $format = [ qw(custnum agent_custid paid payinfo invnum) ]; my $hashcb = sub { my %hash = @_; my $custnum = $hash{'custnum'}; @@ -1225,19 +1223,11 @@ sub process_batch_import { my $opt = { 'table' => 'cust_pay', 'params' => [ '_date', 'agentnum', 'payby', 'paybatch' ], - 'formats' => { - 'simple-csv' => $format, - 'simple-xls' => $format, - }, - 'format_types' => { - 'simple-csv' => 'csv', - 'simple-xls' => 'xls', - }, - 'default_csv' => 1, - 'format_hash_callbacks' => { - 'simple-csv' => $hashcb, - 'simple-xls' => $hashcb, - }, + #agent_custid isn't a cust_pay field, see hash callback + 'formats' => { 'simple' => [ qw(custnum agent_custid paid payinfo invnum) ] }, + 'format_types' => { 'simple' => '' }, #force infer from file extension + 'default_csv' => 1, #if it's not .xls, it'll read as csv, regardless of extension + 'format_hash_callbacks' => { 'simple' => $hashcb }, 'postinsert_callback' => sub { my $cust_pay = shift; my $cust_main = $cust_pay->cust_main || diff --git a/FS/FS/cust_pay_batch.pm b/FS/FS/cust_pay_batch.pm index a4b4957aa..da003d85e 100644 --- a/FS/FS/cust_pay_batch.pm +++ b/FS/FS/cust_pay_batch.pm @@ -265,15 +265,15 @@ sub approve { my %opt = @_; my $paybatchnum = $new->paybatchnum; my $old = qsearchs('cust_pay_batch', { paybatchnum => $paybatchnum }) - or return "paybatchnum $paybatchnum not found"; + or return "cannot approve, paybatchnum $paybatchnum not found"; # leave these restrictions in place until TD EFT is converted over # to B::BP - return "paybatchnum $paybatchnum already resolved ('".$old->status."')" + return "cannot approve paybatchnum $paybatchnum, already resolved ('".$old->status."')" if $old->status; $new->status('Approved'); my $error = $new->replace($old); if ( $error ) { - return "error updating status of paybatchnum $paybatchnum: $error\n"; + return "error approving paybatchnum $paybatchnum: $error\n"; } my $cust_pay = new FS::cust_pay ( { 'custnum' => $new->custnum, @@ -319,7 +319,7 @@ sub decline { my $paybatchnum = $new->paybatchnum; my $old = qsearchs('cust_pay_batch', { paybatchnum => $paybatchnum }) - or return "paybatchnum $paybatchnum not found"; + or return "cannot decline, paybatchnum $paybatchnum not found"; if ( $old->status ) { # Handle the case where payments are rejected after the batch has been # approved. FS::pay_batch::import_results won't allow results to be @@ -344,9 +344,15 @@ sub decline { } $cust_pay->void($reason); } + elsif ( lc($old->status) eq 'declined' ) { + # batch files from RBC can have multiple lines for one decline + # if this causes problems elsewhere, try hacking pay_batch/RBC.pm instead + return ''; + } else { # normal case: refuse to do anything - return "paybatchnum $paybatchnum already resolved ('".$old->status."')"; + # should never happen...only statuses are approved or declined + return "cannot decline paybatchnum $paybatchnum, already resolved ('".$old->status."')"; } } # !$old->status $new->status('Declined'); @@ -354,7 +360,7 @@ sub decline { $new->failure_status($failure_status); my $error = $new->replace($old); if ( $error ) { - return "error updating status of paybatchnum $paybatchnum: $error\n"; + return "error declining paybatchnum $paybatchnum: $error\n"; } my $due_cust_event = $new->cust_main->due_cust_event( 'eventtable' => 'cust_pay_batch', diff --git a/FS/bin/freeside-cdr-mysql b/FS/bin/freeside-cdr-mysql new file mode 100755 index 000000000..608a8dcc3 --- /dev/null +++ b/FS/bin/freeside-cdr-mysql @@ -0,0 +1,88 @@ +#!/usr/bin/perl + +use strict; +use vars qw( $DEBUG ); +use Date::Parse 'str2time'; +use Date::Format 'time2str'; +use FS::UID qw(adminsuidsetup dbh); +use FS::cdr; +use DBI; +use Getopt::Std; + +my %opt; +getopts('H:U:P:D:T:', \%opt); +my $user = shift or die &usage; + +my $dsn = 'dbi:mysql'; +$dsn .= ":database=$opt{D}" if $opt{D}; +$dsn .= ":host=$opt{H}" if $opt{H}; + +my $mysql = DBI->connect($dsn, $opt{U}, $opt{P}) + or die $DBI::errstr; + +adminsuidsetup $user; + +my $fsdbh = FS::UID::dbh; + +# check for existence of freesidestatus +my $table = $opt{T} || 'cdr'; +my $status = $mysql->selectall_arrayref("SHOW COLUMNS FROM $table WHERE Field = 'freesidestatus'"); +if( ! @$status ) { + print "Adding freesidestatus column...\n"; + $mysql->do("ALTER TABLE $table ADD COLUMN freesidestatus varchar(32)") + or die $mysql->errstr; +} +else { + print "freesidestatus column present\n"; +} + +my @cols = ( qw( +calldate clid src dst dcontext channel lastapp lastdata duration + billsec disposition amaflags accountcode uniqueid userfield) ); +my $sql = 'SELECT '.join(',', @cols). " FROM $table WHERE freesidestatus IS NULL"; +my $sth = $mysql->prepare($sql); +$sth->execute; +print "Importing ".$sth->rows." records...\n"; + +my $cdr_batch = new FS::cdr_batch({ + 'cdrbatch' => 'mysql-import-'. time2str('%Y/%m/%d-%T',time), + }); +my $error = $cdr_batch->insert; +die $error if $error; +my $cdrbatchnum = $cdr_batch->cdrbatchnum; +my $imports = 0; +my $updates = 0; + +my $row; +while ( $row = $sth->fetchrow_hashref ) { + my $cdr = FS::cdr->new($row); + $cdr->startdate(str2time($cdr->calldate)); + $cdr->cdrbatchnum($cdrbatchnum); + my $error = $cdr->insert; + if($error) { + print "failed import: $error\n"; + } + else { + $imports++; + if( $mysql->do("UPDATE cdr SET freesidestatus = 'done' + WHERE calldate = ? AND src = ? AND dst = ?", + undef, + $row->{'calldate'}, + $row->{'src'}, + $row->{'dst'}, + + ) ) { + $updates++; + } + else { + print "failed to set status: ".$mysql->errstr."\n"; + } + } +} +print "Done.\nImported $imports CDRs, marked $updates CDRs as done.\n"; +$mysql->disconnect; + +sub usage { + "Usage: \n cdr-mysql.import\n\t[ -H host ]\n\t-D database\n\t-U user\n\t-P password\n\tfreesideuser\n"; +} + diff --git a/FS/bin/freeside-cdr-thinktel b/FS/bin/freeside-cdr-thinktel new file mode 100755 index 000000000..75a6d925c --- /dev/null +++ b/FS/bin/freeside-cdr-thinktel @@ -0,0 +1,127 @@ +#!/usr/bin/perl + +use strict; +use Getopt::Std; +use Date::Format; +use File::Temp 'tempdir'; +use Net::FTP; +use FS::UID qw(adminsuidsetup datasrc dbh); +use FS::cdr; +use FS::cdr_batch; +use FS::Record qw(qsearch qsearchs); +use Date::Format 'time2str'; +use Date::Parse 'str2time'; + + +### +# parse command line +### + +use vars qw( $opt_d $opt_v $opt_c $opt_s $opt_e $opt_a ); +getopts('dvc:s:e:a'); + +my ($user, $login, $password) = @ARGV; +($user and $login and $password) or die &usage; + +my $dbh = adminsuidsetup $user; +$FS::UID::AutoCommit = 0; + +# index already-downloaded batches +my @previous = qsearch({ + 'table' => 'cdr_batch', + 'hashref' => { 'cdrbatch' => {op=>'like', value=>'thinktel%'} }, + 'order_by' => 'ORDER BY cdrbatch DESC', +}); +my %exists = map {$_->cdrbatch => 1} @previous; + +my $tempdir = tempdir( CLEANUP => !$opt_v ); + +my $format = 'thinktel'; +my $hostname = 'ucontrol.thinktel.ca'; + +my $ftp = Net::FTP->new($hostname, Debug => $opt_d) + or die "Can't connect to $hostname: $@\n"; + +$ftp->login($login, $password) + or die "Login failed: ".$ftp->message."\n"; + +### +# get the file list +### + +warn "Retrieving directory listing\n" if $opt_v; + +$ftp->cwd('/'); +my @files = grep { $_ =~ /MetaSwitch/ } $ftp->ls(); + +warn scalar(@files)." CDR files found.\n" if $opt_v; +# apply date range +if ( $opt_a ) { + my $most_recent = $previous[0]; + if ($most_recent) { + if ($most_recent->cdrbatch =~ /^thinktel-(\d+)/) { + my $date = $1; + warn "limiting to dates > $date (from most recent batch)\n" if $opt_v; + @files = grep { /^(\d+)_/ && $1 > $date } @files; + } + } # else download them all +} +if ( $opt_s ) { + # start date + # normalize date format + $opt_s = time2str('%Y%m%d', str2time($opt_s)) if $opt_s =~ /\D/; + warn "limiting to dates > $opt_s\n" if $opt_v; + @files= grep { /^(\d+)_/ && $1 >= $opt_s } @files; +} +if ( $opt_e ) { + # end date + $opt_e = time2str('%Y%m%d', str2time($opt_e)) if $opt_e =~ /\D/; + warn "limiting to dates < $opt_e\n" if $opt_v; + @files= grep { /^(\d+)_/ && $1 < $opt_e } @files; +} +warn scalar(@files) ." files to be downloaded to '$tempdir'\n" if $opt_v; +foreach my $file (@files) { + + warn "downloading $file\n" if $opt_v; + $ftp->get($file, "$tempdir/$file"); + warn "processing $file\n" if $opt_v; + + my $batchname = "$format-$file"; + if ($exists{$batchname}) { + warn "already imported $file\n"; + next; + } + my $import_options = { + 'file' => "$tempdir/$file", + 'format' => $format, + 'batch_namevalue' => $batchname, + 'empty_ok' => 1, + }; + $import_options->{'cdrtypenum'} = $opt_c if $opt_c; + + my $error = FS::cdr::batch_import($import_options); + + if ( $error ) { + die "error processing $file: $error\n"; + } +} +warn "finished\n" if $opt_v; +$dbh->commit; + +### +# subs +### + +sub usage { + "Usage: \n freeside-cdr-thinktel [ options ] user login password + Options: + -v: be verbose + -d: enable FTP debugging (very noisy) + -c num: apply a cdrtypenum to the imported CDRs + -s date: start date + -e date: end date + -a: automatically choose start date from most recently downloaded batch + +"; +} + diff --git a/FS/bin/freeside-cdr-voip_ms b/FS/bin/freeside-cdr-voip_ms new file mode 100755 index 000000000..9c8dc7952 --- /dev/null +++ b/FS/bin/freeside-cdr-voip_ms @@ -0,0 +1,70 @@ +#!/usr/bin/perl + +use strict; +use FS::Misc::Getopt; +use FS::cdr_batch; +use FS::part_export; +use FS::Record qw(qsearch qsearchs dbh); +use Date::Format 'time2str'; + +### +# parse command line +### + +our %opt; +getopts(''); + +$FS::UID::AutoCommit = 0; + +my @exports = qsearch('part_export', { exporttype => 'voip_ms' }); +if (!@exports) { + die "There are no voip.ms exports configured.\n"; +} + +foreach my $part_export (@exports) { + debug "Account #".$part_export->option('account'); + + if (!$opt{start}) { + # find the most recently downloaded batch + my $exportnum = $part_export->exportnum; + my $most_recent = qsearchs({ + 'table' => 'cdr_batch', + 'hashref' => { 'cdrbatch' => {op=>'like', + value=>'voip_ms-' . $exportnum . '-%'} + }, + 'order_by' => 'ORDER BY _date DESC LIMIT 1', + }); + if ( $most_recent ) { + $most_recent->cdrbatch =~ /-(\d+)$/; # extract the end timestamp + $opt{start} = $1; + debug "Downloading records since most recent batch: ". + time2str('%Y-%m-%d', $opt{start}); + } else { + $opt{start} = 1262332800; + debug "Downloading records since January 2010."; + } + } + + $opt{end} ||= time; + + my $error_or_batch = $part_export->import_cdrs( $opt{start}, $opt{end} ); + if ( ref $error_or_batch ) { + debug "Created batch #".$error_or_batch->cdrbatchnum; + dbh->commit; + } elsif ( $error_or_batch ) { + warn $error_or_batch; + dbh->rollback; + } else { + debug "No CDRs found." + } +} + +sub usage { + "Usage: \n freeside-cdr-voip_ms [ options ] user + Options: + -v: be verbose + -s date: start date (defaults to the most recent batch date) + -e date: end date +"; +} + |