RT#38278: Removing duplicate CDR entries prior to billing

2 files changed, 40 insertions, 6 deletions

diff --git a/FS/FS/Conf.pm b/FS/FS/Conf.pm
index 20f2cc4..d361031 100644
--- a/FS/FS/Conf.pm
+++ b/FS/FS/Conf.pm
@@ -4995,6 +4995,13 @@ and customer address. Include units.',
 #  },
 
   {
+    'key'         => 'cdr-skip_duplicate_rewrite',
+    'section'     => 'telephony',
+    'description' => 'Use the freeside-cdrrewrited daemon to prevent billing CDRs with a src, dst and calldate identical to an existing CDR',
+    'type'        => 'checkbox',
+  },
+
+  {
     'key'         => 'cdr-charged_party_rewrite',
     'section'     => 'telephony',
     'description' => 'Do charged party rewriting in the freeside-cdrrewrited daemon; useful if CDRs are being dropped off directly in the database and require special charged_party processing such as cdr-charged_party-accountcode or cdr-charged_party-truncate*.',
diff --git a/FS/bin/freeside-cdrrewrited b/FS/bin/freeside-cdrrewrited
index 0087590..1745d67 100644
--- a/FS/bin/freeside-cdrrewrited
+++ b/FS/bin/freeside-cdrrewrited
@@ -4,7 +4,7 @@ use strict;
 use vars qw( $conf );
 use FS::Daemon ':all'; #daemonize1 drop_root daemonize2 myexit logfile sig*
 use FS::UID qw( adminsuidsetup );
-use FS::Record qw( qsearch qsearchs );
+use FS::Record qw( qsearch qsearchs dbh );
 #use FS::cdr;
 #use FS::cust_pkg;
 #use FS::queue;
@@ -24,12 +24,12 @@ daemonize2();
 
 $conf = new FS::Conf;
 
-die "not running; cdr-asterisk_forward_rewrite, cdr-charged_party_rewrite ".
-    " and cdr-taqua-accountcode_rewrite conf options are all off\n"
+die "not running; relevant conf options are all off\n"
   unless _shouldrun();
 
 #--
 
+#used for taqua
 my %sessionnum_unmatch = ();
 my $sessionnum_retry = 4 * 60 * 60; # 4 hours
 my $sessionnum_giveup = 4 * 24 * 60 * 60; # 4 days
@@ -45,20 +45,25 @@ while (1) {
   # instead of just doing this search like normal CDRs
 
   #hmm :/
+  #used only by taqua, should have no effect otherwise
   my @recent = grep { ($sessionnum_unmatch{$_} + $sessionnum_retry) > time }
                  keys %sessionnum_unmatch;
   my $extra_sql = scalar(@recent)
                     ? ' AND acctid NOT IN ('. join(',', @recent). ') '
                     : '';
 
+  #order matters for removing dupes--only the first is preserved
+  $extra_sql .= ' ORDER BY acctid '
+    if $conf->exists('cdr-skip_duplicate_rewrite');
+
   my $found = 0;
-  my %skip = ();
+  my %skip = (); #used only by taqua
   my %warning = ();
 
   foreach my $cdr ( 
     qsearch( {
       'table'     => 'cdr',
-      'extra_sql' => 'FOR UPDATE',
+      'extra_sql' => 'FOR UPDATE', #XXX overwritten by opt below...would fixing this break anything?
       'hashref'   => {},
       'extra_sql' => 'WHERE freesidestatus IS NULL '.
                      ' AND freesiderewritestatus IS NULL '.
@@ -67,11 +72,27 @@ while (1) {
     } )
   ) {
 
-    next if $skip{$cdr->acctid};
+    next if $skip{$cdr->acctid}; #used only by taqua
 
     $found = 1;
     my @status = ();
 
+    if ($conf->exists('cdr-skip_duplicate_rewrite')) {
+      #qsearch can't handle timestamp type of calldate
+      my $sth = dbh->prepare(
+        'SELECT 1 FROM cdr WHERE src=? AND dst=? AND calldate=? AND acctid < ? LIMIT 1'
+      ) or die dbh->errstr;
+      $sth->execute($cdr->src,$cdr->dst,$cdr->calldate,$cdr->acctid) or die $sth->errstr;
+      my $isdup = $sth->fetchrow_hashref;
+      $sth->finish;
+      if ($isdup) {
+        #we only act on this cdr, not touching previous dupes
+        #if a dupe somehow creeped in previously, too late to fix it
+        $cdr->freesidestatus('done'); #prevent it from being billed
+        push(@status,'duplicate');
+      }
+    }
+
     if ( $conf->exists('cdr-asterisk_forward_rewrite')
          && $cdr->dstchannel =~ /^Local\/(\d+)/i && $1 ne $cdr->dst
        )
@@ -240,6 +261,7 @@ sub _shouldrun {
   || $conf->exists('cdr-taqua-accountcode_rewrite')
   || $conf->exists('cdr-taqua-callerid_rewrite')
   || $conf->exists('cdr-intl_to_domestic_rewrite')
+  || $conf->exists('cdr-skip_duplicate_rewrite')
   || 0
   ;
 }
@@ -263,6 +285,11 @@ of the following config options are enabled:
 
 =over 4
 
+=item cdr-skip_duplicate_rewrite
+
+Marks as 'done' (prevents billing for) any CDRs with 
+a src, dst and calldate identical to an existing CDR
+
 =item cdr-asterisk_australia_rewrite
 
 Classifies Australian numbers as domestic, mobile, tollfree, international, or