1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
|
#!/usr/bin/perl
use strict;
use Getopt::Std;
use Date::Format;
use File::Temp 'tempdir';
use Net::SFTP::Foreign::Compat;
use FS::UID qw(adminsuidsetup datasrc dbh);
use FS::cdr;
use FS::cdr_batch;
use FS::Record qw(qsearch qsearchs);
use Date::Format 'time2str';
use Date::Parse 'str2time';
###
# parse command line
###
use vars qw( $opt_d $opt_v $opt_c $opt_s $opt_e $opt_a );
getopts('dvc:s:e:a');
my ($user, $login, $password) = @ARGV;
($user and $login and $password) or die &usage;
my $dbh = adminsuidsetup $user;
$FS::UID::AutoCommit = 0;
# index already-downloaded batches
my @previous = qsearch({
'table' => 'cdr_batch',
'hashref' => { 'cdrbatch' => {op=>'like', value=>'ani_networks%'} },
'order_by' => 'ORDER BY cdrbatch DESC',
});
my %exists = map {$_->cdrbatch => 1} @previous;
my $format = 'ani_networks';
my $host = 'arkftp.aninetworks.com';
###
# get the file list
###
warn "Retrieving directory listing\n" if $opt_v;
my $sftp = sftp();
## get the current working dir
my $cwd = $sftp->cwd;
## switch to CDR dir
$sftp->setcwd($cwd . '/CDR') or die "can't chdir to $cwd/CDR\n";
my $ls = $sftp->ls('.', wanted => qr/^UYM.*.zip$/i, names_only =>1 );
my @files = @$ls;
warn scalar(@files)." CDR files found.\n" if $opt_v;
# apply date range from last downloaded batch.
if ( $opt_a ) {
my $most_recent = $previous[0];
if ($most_recent) {
if ($most_recent->cdrbatch =~ /^*Daily_(\d+)_/) {
my $date = $1;
warn "limiting to dates >= $date (from most recent batch)\n" if $opt_v;
@files = grep { /^*Daily_(\d+)_/ && $1 >= $date } @files;
}
}
}
# apply a start date if given
if ( $opt_s ) {
# normalize date format
$opt_s = time2str('%Y%m%d', str2time($opt_s)) if $opt_s =~ /\D/;
warn "limiting to dates > $opt_s\n" if $opt_v;
@files= grep { /^*Daily_(\d+)_/ && $1 >= $opt_s } @files;
}
# apply a end date if given
if ( $opt_e ) {
# normalize date format
$opt_e = time2str('%Y%m%d', str2time($opt_e)) if $opt_e =~ /\D/;
warn "limiting to dates < $opt_e\n" if $opt_v;
@files= grep { /^*Daily_(\d+)_/ && $1 < $opt_e } @files;
}
warn scalar(@files) ." files to be downloaded\n" if $opt_v;
foreach my $file (@files) {
my $tmpdir = tempdir( CLEANUP => $opt_v );
warn "downloading $file to $tmpdir\n" if $opt_v;
$sftp = sftp();
$sftp->get($file, "$tmpdir/$file");
## extract zip file
if(system ("unzip $tmpdir/$file -d $tmpdir") != 0) {
unlink "$tmpdir/$file";
my $error = "unzip of '$tmpdir/$file' failed\n";
if ( $opt_s ) {
warn $error;
next;
} else {
die $error;
}
}
warn "processing $file\n" if $opt_v;
my $batchname = "$format-$file";
if ($exists{$batchname}) {
warn "already imported $file\n";
next;
}
my $unzipped_file = $file;
$unzipped_file =~ s/.zip/.txt/i;
warn "going to import file $unzipped_file" if $opt_v;
my $import_options = {
'file' => "$tmpdir/$unzipped_file",
'format' => $format,
'batch_namevalue' => $batchname,
'empty_ok' => 1,
};
$import_options->{'cdrtypenum'} = $opt_c if $opt_c;
my $error = FS::cdr::batch_import($import_options);
if ( $error ) {
die "error processing $unzipped_file: $error\n";
}
}
warn "finished\n" if $opt_v;
$dbh->commit;
###
# subs
###
sub usage {
"Usage: \n freeside-cdr-aninetworks-import [ options ] user login password
Options:
-v: be verbose
-d: enable FTP debugging (very noisy)
-c num: apply a cdrtypenum to the imported CDRs
-s date: start date
-e date: end date
-a: automatically choose start date from most recently downloaded batch
";
}
sub sftp {
#reuse connections
return $sftp if $sftp && $sftp->cwd;
my %sftp = ( host => $host,
user => $login,
password => $password,
more => [-o => 'StrictHostKeyChecking no'],
);
$sftp = Net::SFTP::Foreign->new(%sftp);
$sftp->error and die "SFTP connection failed: ". $sftp->error;
$sftp;
}
=head1 NAME
freeside-cdr-aninetworks-import - Download CDR files from a remote server via SFTP
=head1 SYNOPSIS
freeside-cdr-aninetworks-import [ -v ] [ -d ] [ -a ] [ -c cdrtypenum ]
[ -s startdate ] [ -e enddate ] user sftpuser sftppassword
=head1 DESCRIPTION
Command line tool to download CDR files from a remote server via SFTP
and then import them into the database.
-v: be verbose
-d: enable sftp debugging (very noisy)
-a: automatically choose start date from most recently downloaded batch
-c: cdrtypenum to set, defaults to none
-s: if specified, sets a startdate. startdate starts at 00:00:00
-e: if specified, sets a enddate. enddate starts at 00:00:00 so if you wish to include enddate must add one more day.
user: freeside username
sftpuser: sftp user for arkftp.aninetworks.com
sftppassword: password for sftp user
=head1 EXAMPLES
freeside-cdr-aninetworks-import -a <freeside user> <sftp login> <sftp password>
will get all cdr files starting from the day of the last day processed.
freeside-cdr-aninetworks-import -s 20180120 -e 20180121 <freeside user> <sftp login> <sftp password>
will get all cdr files from 01/20/2018
freeside-cdr-aninetworks-import -v -s $(date --date="-1 day" +\%Y\%m\%d) -e $(date +\%Y\%m\%d) <freeside user> <sftp login> <sftp password>
will get all cdr files from yesterday
=head1 BUGS
=head1 SEE ALSO
L<FS::cdr>
=cut
1;
|
