1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
|
#!/usr/bin/perl
use strict;
use Getopt::Std;
use Date::Format;
use File::Temp 'tempdir';
use Net::SFTP::Foreign::Compat;
use FS::UID qw(adminsuidsetup datasrc dbh);
use FS::cdr;
use FS::cdr_batch;
use FS::Record qw(qsearch qsearchs);
use Date::Format 'time2str';
use Date::Parse 'str2time';
###
# parse command line
###
use vars qw( $opt_d $opt_v $opt_c $opt_s $opt_e $opt_a );
getopts('dvc:s:e:a');
my ($user, $login, $password) = @ARGV;
($user and $login and $password) or die &usage;
my $dbh = adminsuidsetup $user;
$FS::UID::AutoCommit = 0;
# index already-downloaded batches
my @previous = qsearch({
'table' => 'cdr_batch',
'hashref' => { 'cdrbatch' => {op=>'like', value=>'ani_networks%'} },
'order_by' => 'ORDER BY cdrbatch DESC',
});
my %exists = map {$_->cdrbatch => 1} @previous;
my $format = 'ani_networks';
my $host = 'arkftp.aninetworks.com';
###
# get the file list
###
warn "Retrieving directory listing\n" if $opt_v;
my $sftp = sftp();
## get the current working dir
my $cwd = $sftp->cwd;
## switch to CDR dir
$sftp->setcwd($cwd . '/CDR') or die "can't chdir to $cwd/CDR\n";
my $ls = $sftp->ls('.', wanted => qr/^UYM.*.zip$/i, names_only =>1 );
my @files = @$ls;
warn scalar(@files)." CDR files found.\n" if $opt_v;
# apply date range
if ( $opt_a ) {
my $most_recent = $previous[0];
if ($most_recent) {
if ($most_recent->cdrbatch =~ /^ani_networks-(\d+)/) {
my $date = $1;
warn "limiting to dates > $date (from most recent batch)\n" if $opt_v;
@files = grep { /^*Daily_(\d+)_/ && $1 > $date } @files;
}
} # else download them all
}
if ( $opt_s ) {
# start date
# normalize date format
$opt_s = time2str('%Y%m%d', str2time($opt_s)) if $opt_s =~ /\D/;
warn "limiting to dates > $opt_s\n" if $opt_v;
@files= grep { /^*Daily_(\d+)_/ && $1 >= $opt_s } @files;
}
if ( $opt_e ) {
# end date
$opt_e = time2str('%Y%m%d', str2time($opt_e)) if $opt_e =~ /\D/;
warn "limiting to dates < $opt_e\n" if $opt_v;
@files= grep { /^*Daily_(\d+)_/ && $1 < $opt_e } @files;
}
warn scalar(@files) ." files to be downloaded\n" if $opt_v;
foreach my $file (@files) {
my $tmpdir = tempdir( CLEANUP => $opt_v );
warn "downloading $file to $tmpdir\n" if $opt_v;
$sftp = sftp();
$sftp->get($file, "$tmpdir/$file");
## extract zip file
if(system ("unzip $tmpdir/$file -d $tmpdir") != 0) {
unlink "$tmpdir/$file";
my $error = "unzip of '$tmpdir/$file' failed\n";
if ( $opt_s ) {
warn $error;
next;
} else {
die $error;
}
}
warn "processing $file\n" if $opt_v;
my $batchname = "$format-$file";
if ($exists{$batchname}) {
warn "already imported $file\n";
next;
}
my $unzipped_file = $file;
$unzipped_file =~ s/.zip/.txt/i;
warn "going to import file $unzipped_file" if $opt_v;
my $import_options = {
'file' => "$tmpdir/$unzipped_file",
'format' => $format,
'batch_namevalue' => $batchname,
'empty_ok' => 1,
};
$import_options->{'cdrtypenum'} = $opt_c if $opt_c;
my $error = FS::cdr::batch_import($import_options);
if ( $error ) {
die "error processing $unzipped_file: $error\n";
}
}
warn "finished\n" if $opt_v;
$dbh->commit;
###
# subs
###
sub usage {
"Usage: \n freeside-cdr-aninetworks-import [ options ] user login password
Options:
-v: be verbose
-d: enable FTP debugging (very noisy)
-c num: apply a cdrtypenum to the imported CDRs
-s date: start date
-e date: end date
-a: automatically choose start date from most recently downloaded batch
";
}
sub sftp {
#reuse connections
return $sftp if $sftp && $sftp->cwd;
my %sftp = ( host => $host,
user => $login,
password => $password,
more => [-o => 'StrictHostKeyChecking no'],
);
$sftp = Net::SFTP::Foreign->new(%sftp);
$sftp->error and die "SFTP connection failed: ". $sftp->error;
$sftp;
}
|
