1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
|
#!/usr/bin/perl
use strict;
use Getopt::Std;
use Date::Format;
use Net::SFTP::Foreign::Compat;
use FS::UID qw(adminsuidsetup datasrc);
use FS::cdr;
### So much false laziness with freeside-cdr-sftp_and_import
### but vocus needs special handling to choose which files to load
###
# parse command line
###
use vars qw( $opt_r $opt_d $opt_v $opt_s );
getopts('r:d:vs');
my $user = shift or die &usage;
adminsuidsetup $user;
# %%%FREESIDE_CACHE%%%
my $cachedir = '%%%FREESIDE_CACHE%%%/cache.'. datasrc. '/cdrs';
mkdir $cachedir unless -d $cachedir;
use vars qw( $servername );
$servername = shift or die &usage;
###
# get the file list
###
warn "Retrieving directory listing\n" if $opt_v;
my $ls;
my $ls_sftp = sftp();
$ls_sftp->setcwd($opt_r) or die "can't chdir to $opt_r\n"
if $opt_r;
$ls = $ls_sftp->ls('.', wanted => qr/^VAP.*\.csv\.gz$/i,
names_only => 1 );
###
# vocus-specific part -- only use highest-numbered file for day
###
my %dates;
foreach my $filename ( @$ls ) {
my ($filepre,$filedate,$iter) = $filename =~ /^(VAP\d+\-)(\d{4}\-\d{2}\-\d{2})(\.\d+)?.csv.gz$/;
unless ($filepre && $filedate) {
die "unparsable filename $filename"; #no warn and skip, might process wrong file for date
}
$iter =~ s/\.// if length($iter);
#not clear if same day can have different initial digits after VAP,
# stated rule is "use the highest-numbered file for a given day",
# so using date as key, but die if iter can't resolve conflict
if (!$dates{$filedate}) {
$dates{$filedate} = {};
} elsif ($dates{$filedate}{'iter'} eq $iter) {
die "duplicate iterators found for $filedate\n"
}
$dates{$filedate}{'files'} ||= [];
push @{$dates{$filedate}{'files'}}, $filename;
# don't actually expect iter of 0, but just in case, 0 trumps undef
if (!defined($dates{$filedate}{'iter'}) or (($iter || 0) > $dates{$filedate}{'iter'})) {
$dates{$filedate}{'iter'} = $iter;
$dates{$filedate}{'pre'} = $filepre;
}
}
###
# import each file
###
foreach my $filedate ( keys %dates ) {
my $filename = $dates{$filedate}{'pre'} . $filedate;
$filename .= '.'.$dates{$filedate}{'iter'}
if defined($dates{$filedate}{'iter'});
$filename .= '.csv.gz';
warn "Downloading $filename\n" if $opt_v;
#get the file
my $sftp = sftp();
$sftp->get($filename, "$cachedir/$filename")
or do {
unlink "$cachedir/$filename";
my $error = "Can't get $filename: ". $sftp->error . "\n";
if ( $opt_s ) {
warn $error;
next;
} else {
die $error;
}
};
warn "Processing $filename\n" if $opt_v;
my $ungziped = $filename;
$ungziped =~ s/\.gz$//;
if(system('gunzip', "$cachedir/$filename") != 0) {
unlink "$cachedir/$filename";
my $error = "gunzip of '$cachedir/$filename' failed\n";
if ( $opt_s ) {
warn $error;
next;
} else {
die $error;
}
}
my $import_options = {
'file' => "$cachedir/$ungziped",
'format' => 'vocus',
'batch_namevalue' => 'vocus-'.$filedate, #should further ensure only one file per date
# 'empty_ok' => 1,
};
my $error = FS::cdr::batch_import($import_options);
if ( $error ) {
unlink "$cachedir/$filename";
unlink "$cachedir/$ungziped";
$error = "Error importing $ungziped: $error\n";
if ( $opt_s ) {
warn $error;
next;
} else {
die $error;
}
} else {
if ( $opt_d ) {
my $timestamp = time2str('%Y-%m-%d', time);
my $sftp = sftp();
foreach my $mfilename (@{$dates{$filedate}{'files'}}) {
warn "Moving $mfilename\n" if $opt_v;
$sftp->rename($mfilename, "$opt_d/$mfilename-$timestamp")
or do {
unlink "$cachedir/$filename";
unlink "$cachedir/$ungziped";
$error = "$mfilename imported, but can't move to $opt_d: ". $sftp->error . "\n";
if ( $opt_s ) {
warn $error;
next;
} else {
die $error;
}
};
}
}
}
unlink "$cachedir/$filename";
unlink "$cachedir/$ungziped";
}
###
# subs
###
sub usage {
"Usage:
freeside-cdr-vocus
[ -r remotefolder ] [ -d donefolder ] [ -v level ]
[ -s ] user [sftpuser@]servername
";
}
use vars qw( $sftp );
sub sftp {
#reuse connections
return $sftp if $sftp && $sftp->cwd;
my %sftp = ( host => $servername );
$sftp = Net::SFTP::Foreign->new(%sftp);
$sftp->error and die "SFTP connection failed: ". $sftp->error;
$sftp;
}
=head1 NAME
freeside-cdr-vocus - Download Vocus CDR files
=head1 SYNOPSIS
freeside-cdr-vocus
[ -r remotefolder ] [ -d donefolder ] [ -v level ]
[ -s ] user [sftpuser@]servername
=head1 DESCRIPTION
Command line tool to download Vocus CDR files from a remote server via SFTP
and then import them into the database. CDR tarrif types need to be
configured as CDR types in freeside. If multiple files for a given day are
found, the one with the highest iterator will be used (though upon successful
import, all existing files for the day will be moved by the -d option.)
-r: if specified, changes into this remote folder before starting
-d: if specified, moves files to the specified folder when done
-v: verbose
-s: Warn and skip files which could not be imported rather than abort
user: freeside username
[sftpuser@]servername: remote server
=head1 BUGS
=head1 SEE ALSO
L<FS::cdr>
=cut
1;
|
