6 use File::Temp 'tempdir';
7 use Net::SFTP::Foreign::Compat;
8 use FS::UID qw(adminsuidsetup datasrc dbh);
11 use FS::Record qw(qsearch qsearchs);
12 use Date::Format 'time2str';
13 use Date::Parse 'str2time';
20 use vars qw( $opt_d $opt_v $opt_c $opt_s $opt_e $opt_a );
23 my ($user, $login, $password) = @ARGV;
24 ($user and $login and $password) or die &usage;
26 my $dbh = adminsuidsetup $user;
27 $FS::UID::AutoCommit = 0;
29 # index already-downloaded batches
30 my @previous = qsearch({
31 'table' => 'cdr_batch',
32 'hashref' => { 'cdrbatch' => {op=>'like', value=>'ani_networks%'} },
33 'order_by' => 'ORDER BY cdrbatch DESC',
35 my %exists = map {$_->cdrbatch => 1} @previous;
37 my $format = 'ani_networks';
38 my $host = 'arkftp.aninetworks.com';
44 warn "Retrieving directory listing\n" if $opt_v;
48 ## get the current working dir
52 $sftp->setcwd($cwd . '/CDR') or die "can't chdir to $cwd/CDR\n";
54 my $ls = $sftp->ls('.', wanted => qr/^UYM.*.zip$/i, names_only =>1 );
57 warn scalar(@files)." CDR files found.\n" if $opt_v;
58 # apply date range from last downloaded batch.
60 my $most_recent = $previous[0];
62 if ($most_recent->cdrbatch =~ /^*Daily_(\d+)_/) {
64 warn "limiting to dates >= $date (from most recent batch)\n" if $opt_v;
65 @files = grep { /^*Daily_(\d+)_/ && $1 >= $date } @files;
70 # apply a start date if given
72 # normalize date format
73 $opt_s = time2str('%Y%m%d', str2time($opt_s)) if $opt_s =~ /\D/;
74 warn "limiting to dates > $opt_s\n" if $opt_v;
75 @files= grep { /^*Daily_(\d+)_/ && $1 >= $opt_s } @files;
78 # apply a end date if given
80 # normalize date format
81 $opt_e = time2str('%Y%m%d', str2time($opt_e)) if $opt_e =~ /\D/;
82 warn "limiting to dates < $opt_e\n" if $opt_v;
83 @files= grep { /^*Daily_(\d+)_/ && $1 < $opt_e } @files;
86 warn scalar(@files) ." files to be downloaded\n" if $opt_v;
87 foreach my $file (@files) {
89 my $tmpdir = tempdir( CLEANUP => $opt_v );
91 warn "downloading $file to $tmpdir\n" if $opt_v;
93 $sftp->get($file, "$tmpdir/$file");
96 if(system ("unzip $tmpdir/$file -d $tmpdir") != 0) {
97 unlink "$tmpdir/$file";
98 my $error = "unzip of '$tmpdir/$file' failed\n";
107 warn "processing $file\n" if $opt_v;
109 my $batchname = "$format-$file";
110 if ($exists{$batchname}) {
111 warn "already imported $file\n";
115 my $unzipped_file = $file;
116 $unzipped_file =~ s/.zip/.txt/i;
118 warn "going to import file $unzipped_file" if $opt_v;
120 my $import_options = {
121 'file' => "$tmpdir/$unzipped_file",
123 'batch_namevalue' => $batchname,
126 $import_options->{'cdrtypenum'} = $opt_c if $opt_c;
128 my $error = FS::cdr::batch_import($import_options);
131 die "error processing $unzipped_file: $error\n";
134 warn "finished\n" if $opt_v;
142 "Usage: \n freeside-cdr-aninetworks-import [ options ] user login password
145 -d: enable FTP debugging (very noisy)
146 -c num: apply a cdrtypenum to the imported CDRs
149 -a: automatically choose start date from most recently downloaded batch
157 return $sftp if $sftp && $sftp->cwd;
159 my %sftp = ( host => $host,
161 password => $password,
162 more => [-o => 'StrictHostKeyChecking no'],
165 $sftp = Net::SFTP::Foreign->new(%sftp);
166 $sftp->error and die "SFTP connection failed: ". $sftp->error;
173 freeside-cdr-aninetworks-import - Download CDR files from a remote server via SFTP
177 freeside-cdr-aninetworks-import [ -v ] [ -d ] [ -a ] [ -c cdrtypenum ]
178 [ -s startdate ] [ -e enddate ] user sftpuser sftppassword
182 Command line tool to download CDR files from a remote server via SFTP
183 and then import them into the database.
187 -d: enable sftp debugging (very noisy)
189 -a: automatically choose start date from most recently downloaded batch
191 -c: cdrtypenum to set, defaults to none
193 -s: if specified, sets a startdate. startdate starts at 00:00:00
195 -e: if specified, sets a enddate. enddate starts at 00:00:00 so if you wish to include enddate must add one more day.
197 user: freeside username
199 sftpuser: sftp user for arkftp.aninetworks.com
201 sftppassword: password for sftp user
205 freeside-cdr-aninetworks-import -a <freeside user> <sftp login> <sftp password>
206 will get all cdr files starting from the day of the last day processed.
208 freeside-cdr-aninetworks-import -s 20180120 -e 20180121 <freeside user> <sftp login> <sftp password>
209 will get all cdr files from 01/20/2018
211 freeside-cdr-aninetworks-import -v -s $(date --date="-1 day" +\%Y\%m\%d) -e $(date +\%Y\%m\%d) <freeside user> <sftp login> <sftp password>
212 will get all cdr files from yesterday