Development

Navigation

← Previous Changeset
Next Changeset →

Changeset 1338

Timestamp:

09/01/08 20:58:50 (3 months ago)

Author:

scitech

Message:

Ispcp-apache-logger is crashing. Crontab entry for static awstats scripts broken. Partial solve for #1232 domain_traffic becomes huge and slows down server.

Files:

trunk/CHANGELOG (modified) (1 diff)
trunk/configs/centos/apache/httpd.conf (modified) (1 diff)
trunk/configs/debian/apache/httpd.conf (modified) (1 diff)
trunk/configs/fedora/apache/httpd.conf (modified) (1 diff)
trunk/configs/freebsd/apache/httpd.conf (modified) (1 diff)
trunk/configs/gentoo/apache/httpd.conf (modified) (1 diff)
trunk/configs/openbsd/apache/httpd.conf (modified) (1 diff)
trunk/configs/opensuse/apache/httpd.conf (modified) (1 diff)
trunk/configs/redhat/apache/httpd.conf (modified) (1 diff)
trunk/configs/ubuntu/apache/httpd.conf (modified) (1 diff)
trunk/engine/ispcp-apache-logger (modified) (7 diffs)
trunk/engine/ispcp-dmn-mngr (modified) (2 diffs)
trunk/engine/traffic/ispcp-vrl-traff (modified) (20 diffs)

Legend:

: Unmodified
: Added
: Removed
: Modified
: Copied
: Moved

trunk/CHANGELOG

r1337	r1338
1	1	ispCP ω 1.0.0 Changelog
2	2	~~~~~~~~~~~~~~~~~~~~~~~~~~~
	3	2008-09-01 Daniel Andreca
	4	- ENGINE:
	5	* ispcp-apache-logger is crashing
	6	* crontab entry for static awstats scripts broken
	7	* partial solve for #1232 domain_traffic becomes huge and slows down server.
	8
3	9	2008-08-29 Daniel Andreca
4	10	- ENGINE:

trunk/configs/centos/apache/httpd.conf

r1329	r1338
16	16
17	17	LogFormat "%B" traff
18		LogFormat "%v %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ispcplog
	18	LogFormat "%v %b %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ispcplog
19	19
20	20	CustomLog "\| /var/www/ispcp/engine/ispcp-apache-logger" ispcplog

trunk/configs/debian/apache/httpd.conf

r1329	r1338
16	16
17	17	LogFormat "%B" traff
18		LogFormat "%v %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ispcplog
	18	LogFormat "%v %b %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ispcplog
19	19
20	20	CustomLog "\| /var/www/ispcp/engine/ispcp-apache-logger" ispcplog

trunk/configs/fedora/apache/httpd.conf

r1329	r1338
16	16
17	17	LogFormat "%B" traff
18		LogFormat "%v %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ispcplog
	18	LogFormat "%v %b %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ispcplog
19	19
20	20	CustomLog "\| /var/www/ispcp/engine/ispcp-apache-logger" ispcplog

trunk/configs/freebsd/apache/httpd.conf

r1329	r1338
16	16
17	17	LogFormat "%B" traff
18		LogFormat "%v %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ispcplog
	18	LogFormat "%v %b %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ispcplog
19	19
20	20	CustomLog "\| /usr/local/www/ispcp/engine/ispcp-apache-logger" ispcplog

trunk/configs/gentoo/apache/httpd.conf

r1329	r1338
16	16
17	17	LogFormat "%B" traff
18		LogFormat "%v %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ispcplog
	18	LogFormat "%v %b %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ispcplog
19	19
20	20	CustomLog "\| /var/www/ispcp/engine/ispcp-apache-logger" ispcplog

trunk/configs/openbsd/apache/httpd.conf

r1329	r1338
16	16
17	17	LogFormat "%B" traff
18		LogFormat "%v %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ispcplog
	18	LogFormat "%v %b %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ispcplog
19	19
20	20	CustomLog "\| /usr/local/www/apache22/ispcp/engine/ispcp-apache-logger" ispcplog

trunk/configs/opensuse/apache/httpd.conf

r1329	r1338
16	16
17	17	LogFormat "%B" traff
18		LogFormat "%v %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ispcplog
	18	LogFormat "%v %b %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ispcplog
19	19
20	20	CustomLog "\| /srv/www/ispcp/engine/ispcp-apache-logger" ispcplog

trunk/configs/redhat/apache/httpd.conf

r1329	r1338
16	16
17	17	LogFormat "%B" traff
18		LogFormat "%v %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ispcplog
	18	LogFormat "%v %b %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ispcplog
19	19
20	20	CustomLog "\| /var/www/ispcp/engine/ispcp-apache-logger" ispcplog

trunk/configs/ubuntu/apache/httpd.conf

r1329	r1338
16	16
17	17	LogFormat "%B" traff
18		LogFormat "%v %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ispcplog
	18	LogFormat "%v %b %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ispcplog
19	19
20	20	CustomLog "\| /var/www/ispcp/engine/ispcp-apache-logger" ispcplog

trunk/engine/ispcp-apache-logger

r1336	r1338
66	66	sub ErrorLogs();
67	67
68		open (STDERR, ">>", "$main::cfg{'LOG_DIR'}/ispcp-apache-logger.stderr") or die ( "Can't redirect STDERR" );;
	68	open (STDERR, ">", "$main::cfg{'LOG_DIR'}/ispcp-apache-logger.stderr") or die ( "Can't redirect STDERR" );;
69	69	STDERR->autoflush(1);
70		open (STDOUT, ">>", "$main::cfg{'LOG_DIR'}/ispcp-apache-logger.stdout") or die ( "Can't redirect STDOUT" );;
	70	open (STDOUT, ">", "$main::cfg{'LOG_DIR'}/ispcp-apache-logger.stdout") or die ( "Can't redirect STDOUT" );;
71	71	STDOUT->autoflush(1);
72	72
…	…
80	80	while ( my $log_line = <STDIN> ) {
81	81	my $vhost = 'default';
82
	82
83	83	if($log_line =~ m/($main::cfg{'APACHE_WWW_DIR'})/){
84	84	($vhost) = $log_line =~ m/$main::cfg{'APACHE_WWW_DIR'}\/([^\/]*)/;
…	…
88	88	}
89	89	}
90
	90
91	91	if ( $vhost eq 'master' ){
92	92	$vhost='default';
…	…
103	103	}
104	104	}
105
	105
106	106	checkFileExists($vhost, \%error_logs, $main::cfg{'APACHE_USERS_LOG_DIR'},"-error.log",$force_open);
107	107	print { $error_logs{$vhost} } $log_line;
…	…
110	110	sub DefaultLogs(){
111	111	while ( my $log_line = <STDIN> ) {
112		my ($vhost, $rhost, $rlog, $user, $date, $url_with_method, $status, $size, $referrer, $agent) = $log_line =~
113		m/^(\S+) (\S+) ([^ ]?) ([^ ]?) \[(\S+ \+\d{4})\] "(.?)" (\d{3}) (\d+\|-) "(.?)" "([^"]+)"$/;
	112	my ($vhost, $size, $line) = $log_line =~ m/^(\S+) (\d+\|-) (.*)$/s;
114	113	if(!defined($vhost)){print "Trouble line:\n\t$log_line\n"; return 0;}
115	114	if(!defined($size)){print "Trouble line:\n\t$log_line\n"; return 0;}
…	…
119	118	$vhost = $1;
120	119	$vhost = 'default' unless $vhost;
121
	120
122	121	my $force_open=0;
123	122	if ( !$combined_logs{$vhost} \|\| !$traff_logs{$vhost} \|\| !$access_logs{$vhost} ){
…	…
140	139	}
141	140	}
142
	141
143	142	checkFileExists($vhost, \%combined_logs, $main::cfg{'APACHE_LOG_DIR'},"-combined.log",$force_open);
144	143	checkFileExists($vhost, \%traff_logs, $main::cfg{'APACHE_LOG_DIR'},"-traf.log",$force_open);
145	144	checkFileExists($vhost, \%access_logs, $main::cfg{'APACHE_USERS_LOG_DIR'},"-access.log",$force_open);
146
147		$log_line ~~=~ s/^\S*\s+//o~~;
148
	145
	146	$log_line=$line;
	147
149	148	print { $combined_logs{$vhost} } $log_line;
150	149	print { $access_logs{$vhost} } $log_line;

trunk/engine/ispcp-dmn-mngr

r1319	r1338
3006	3006	my $cronjob_hour = '3';
3007	3007	my $conf_dir = $main::cfg{'CONF_DIR'};
	3008	my $virtual_dir = $main::cfg{'APACHE_WWW_DIR'};
3008	3009	my $tpl_dir = "$conf_dir/cron.d/parts";
3009	3010	my $working_dir = "$conf_dir/cron.d/working";
…	…
3105	3106	'{AWSTATS_ENGINE_DIR}' => $awstats_engine_dir,
3106	3107	'{AWSTATS_WEB_DIR}' => $awstats_web_dir,
3107		'{USER_LANG}' => $user_lang
	3108	'{USER_LANG}' => $user_lang,
	3109	'{APACHE_WWW_DIR}' => $virtual_dir
3108	3110	);
3109	3111	(

trunk/engine/traffic/ispcp-vrl-traff

r1336	r1338
51	51	sub zero_traffic_entries {
52	52
53		my ($key, $value) = (undef, undef);
54
55		push_el(\@main::el, 'zero_traffic_entries()', 'Starting...');
56
57		while (($key, $value) = each %main::domain_name_id) {
58
59		$main::ftp_traffic{$key} = 0;
60
61		$main::smtp_traffic{$key} = 0;
62
63		$main::po_traffic{$key} = 0;
64
65		}
66
67		while (($key, $value) = each %main::als_name_id) {
68
69		$main::ftp_traffic{$key} = 0;
70
71		$main::smtp_traffic{$key} = 0;
72
73		$main::po_traffic{$key} = 0;
74
75		}
76
77		push_el(\@main::el, 'zero_traffic_entries()', 'Ending...');
78
79		return 0;
	53	my ($key, $value) = (undef, undef);
	54
	55	push_el(\@main::el, 'zero_traffic_entries()', 'Starting...');
	56
	57	while (($key, $value) = each %main::domain_name_id) {
	58
	59	$main::ftp_traffic{$key} = 0;
	60
	61	$main::smtp_traffic{$key} = 0;
	62
	63	$main::po_traffic{$key} = 0;
	64
	65	}
	66
	67	while (($key, $value) = each %main::als_name_id) {
	68
	69	$main::ftp_traffic{$key} = 0;
	70
	71	$main::smtp_traffic{$key} = 0;
	72
	73	$main::po_traffic{$key} = 0;
	74
	75	}
	76
	77	push_el(\@main::el, 'zero_traffic_entries()', 'Ending...');
	78
	79	return 0;
80	80
81	81	}
…	…
83	83	sub diff_command {
84	84
85		my ($cmd) = @_;
86
87		push_el(\@main::el, 'diff_command()', 'Starting...');
88
89		my $result = system($cmd);
90
91		~~my $exit_value~~ = $? >> 8;
92
93		~~my $signal_num~~ = $? & 127;
94
95		my $dumped_core = $? & 128;
96
97		if ($exit_value == 2) {
98
99		push_el(\@main::el, 'diff_command()', "ERORR: '$cmd' returned '$exit_value' !");
100
101		return $exit_value;
102
103		}
104
105		push_el(\@main::el, 'diff_command()', 'Ending...');
106
107		return $exit_value;
	85	my ($cmd) = @_;
	86
	87	push_el(\@main::el, 'diff_command()', 'Starting...');
	88
	89	my $result = system($cmd);
	90
	91	my $exit_value = $? >> 8;
	92
	93	my $signal_num = $? & 127;
	94
	95	my $dumped_core = $? & 128;
	96
	97	if ($exit_value == 2) {
	98
	99	push_el(\@main::el, 'diff_command()', "ERORR: '$cmd' returned '$exit_value' !");
	100
	101	return $exit_value;
	102
	103	}
	104
	105	push_el(\@main::el, 'diff_command()', 'Ending...');
	106
	107	return $exit_value;
108	108
109	109	}
…	…
111	111	sub gen_log_file {
112	112
113		my ($fname, $src_dir, $dest_dir) = @_;
114
115		push_el(\@main::el, 'gen_log_file()', 'Starting...');
116
117		my $rs = undef;
118
119		$fname =~ s/\/*//i;
120
121		my $src_file = "$src_dir/$fname";
122
123		my $dest_file = "$dest_dir/".basename($fname);
124
125		my $dest_file_prev = "$dest_dir/".basename($fname).".prev";
126
127		my $log_file = $dest_file;
128
129		if(! -e $src_file){
130		push_el(\@main::el, 'gen_log_file()', "WARNING: File '$src_file' do not exists !");
131		return (0, '_no_');
132		}
133
134		$rs = sys_command("$main::cfg{'CMD_CP'} $src_file $dest_file");
135
136		return ($rs, '') if ($rs != 0);
	113	my ($fname, $src_dir, $dest_dir) = @_;
	114
	115	push_el(\@main::el, 'gen_log_file()', 'Starting...');
	116
	117	my $rs = undef;
	118
	119	$fname =~ s/\/*//i;
	120
	121	my $src_file = "$src_dir/$fname";
	122
	123	my $dest_file = "$dest_dir/".basename($fname);
	124
	125	my $dest_file_prev = "$dest_dir/".basename($fname).".prev";
	126
	127	my $log_file = $dest_file;
	128
	129	if(! -e $src_file){
	130
	131	push_el(\@main::el, 'gen_log_file()', "WARNING: File '$src_file' do not exists !");
	132
	133	return (0, '_no_');
	134
	135	}
	136
	137	$rs = sys_command("$main::cfg{'CMD_CP'} $src_file $dest_file");
	138
	139	return ($rs, '') if ($rs != 0);
137	140
138	141	if (-e $dest_file_prev) {
139	142
140		if (diff_command("$main::cfg{'CMD_DIFF'} -a -q $dest_file $dest_file_prev") == 0) {
141
142		$rs = del_file($dest_file);
143
144		return ($rs, '') if ($rs != 0);
145
146		return (0, '_no_');
147
148		}
149
150		$log_file = $dest_file.".diff";
151
152		diff_command("$main::cfg{'CMD_DIFF'} -a $dest_file $dest_file_prev \| $main::cfg{'CMD_EGREP'} '^<' \| $main::cfg{'CMD_SED'} 's/^< //' 1> $log_file");
153
154		$rs = sys_command("$main::cfg{'CMD_MV'} $dest_file $dest_file_prev");
155
156		return ($rs, '') if ($rs != 0);
157
158		} else {
159
160		$rs = sys_command("$main::cfg{'CMD_CP'} $dest_file $dest_file_prev");
161
162		return ($rs, '') if ($rs != 0);
163
164		}
165
166		push_el(\@main::el, 'gen_log_file()', 'Ending...');
167
168		return (0, $log_file);
	143	if (diff_command("$main::cfg{'CMD_DIFF'} -a -q $dest_file $dest_file_prev") == 0) {
	144
	145	$rs = del_file($dest_file);
	146
	147	return ($rs, '') if ($rs != 0);
	148
	149	return (0, '_no_');
	150
	151	}
	152
	153	$log_file = $dest_file.".diff";
	154
	155	diff_command("$main::cfg{'CMD_DIFF'} -a $dest_file $dest_file_prev \| $main::cfg{'CMD_EGREP'} '^<' \| $main::cfg{'CMD_SED'} 's/^< //' 1> $log_file");
	156
	157	$rs = sys_command("$main::cfg{'CMD_MV'} $dest_file $dest_file_prev");
	158
	159	return ($rs, '') if ($rs != 0);
	160
	161	} else {
	162
	163	$rs = sys_command("$main::cfg{'CMD_CP'} $dest_file $dest_file_prev");
	164
	165	return ($rs, '') if ($rs != 0);
	166
	167	}
	168
	169	push_el(\@main::el, 'gen_log_file()', 'Ending...');
	170
	171	return (0, $log_file);
	172	}
	173
	174	sub collect_http_traffic_from_log {
	175
	176	my ($key, $value) = @_;
	177
	178	my ($rs, $rdata, $timestamp, $sum, $rows, $sql) = (undef, undef, time(), 0, undef, undef);
	179
	180	# timestamp has floor down for the last half'n our to make timestamps in
	181	# server_traffic and domain_traffic the same
	182	$timestamp = $timestamp - ($timestamp % 1800);
	183
	184	my $domain_log = "$key-traf.log";
	185
	186	($rs, $rdata) = gen_log_file($domain_log, $main::cfg{'APACHE_LOG_DIR'}, $main::cfg{'LOG_DIR'});
	187
	188	return ($rs, '') if ($rs != 0);
	189
	190	$sum = 0;
	191
	192	if ($rdata ne '_no_') {
	193
	194	my $res = open(F, '<', $rdata);
	195
	196	if (!defined($res)) {
	197	push_el(\@main::el, 'collect_http_traffic()', "ERROR: Can't open '$rdata' for reading !");
	198	return (-1, '');
	199	}
	200
	201	while (<F>) {
	202	$sum += $1 if ($_ =~ /(\d+)\n$/);
	203	}
	204
	205	close(F);
	206
	207	$rs = del_file($rdata);
	208
	209	return ($rs, '') if ($rs != 0);
	210
	211	}
	212
	213	if($sum != 0){
	214
	215	$sql="select COUNT(`domain_id`) from domain_traffic where `domain_id`='$value' and `dtraff_time`>=UNIX_TIMESTAMP(CURRENT_DATE())";
	216
	217	($rs, $rows) = doSQL($sql);
	218
	219	return $rs if ($rs != 0);
	220
	221	if($$rows[0][0] == 0){
	222
	223	$sql = "insert into `domain_traffic` (domain_id, dtraff_time, dtraff_web, dtraff_ftp, dtraff_mail, dtraff_pop) values ('$value', '$timestamp', $sum, '0', '0', '0');";
	224
	225	} else {
	226
	227	$sql = "update `domain_traffic` set `dtraff_web`=`dtraff_web`+$sum, `dtraff_time`='$timestamp', `correction`=0 where `domain_id`='$value' and `dtraff_time`>=UNIX_TIMESTAMP(CURRENT_DATE()) LIMIT 1";
	228
	229	}
	230
	231	($rs, $rdata) = doSQL($sql);
	232
	233	return ($rs, '') if ($rs != 0);
	234
	235	}
	236
	237	return (0, $timestamp);
	238
169	239	}
170	240
171	241	sub collect_http_traffic {
172	242
173		my ($key, $value, $timestamp, $sum) = (undef, undef, time(), 0);
174
175		# timestamp has floor down for the last half'n our to make timestamps in
176		# server_traffic and domain_traffic the same
177		$timestamp = $timestamp - ($timestamp % 1800);
178
179		my ($rs, $rdata) = (undef, undef);
180
181		push_el(\@main::el, 'collect_http_traffic()', 'Starting...');
182
183		while (($key,$value) = each %main::domain_name_id) {
184
185		my $domain_log = "$key-traf.log";
186
187		($rs, $rdata) = gen_log_file($domain_log, $main::cfg{'APACHE_LOG_DIR'}, $main::cfg{'LOG_DIR'});
188
189		return ($rs, '') if ($rs != 0);
190
191		$sum = 0;
192
193		if ($rdata ne '_no_') {
194
195		my $rlog = $rdata;
196
197		my $res = open(F, '<', $rlog);
198
199		if (!defined($res)) {
200		push_el(\@main::el, 'collect_http_traffic()', "ERROR: Can't open '$rlog' for reading !");
201		return (-1, '');
202		}
203
204		while (<F>) {
205		$sum += $1 if ($_ =~ /(\d+)\n$/);
206		}
207
208		close(F);
209
210		$rs = del_file($rlog);
211
212		return ($rs, '') if ($rs != 0);
213
214		}
215
216		my $sql = undef;
217
218		$sql = "insert into domain_traffic (domain_id, dtraff_time, dtraff_web, dtraff_ftp, dtraff_mail, dtraff_pop) values ('$value', '$timestamp', '$sum', '0', '0', '0');";
219
220		($rs, $rdata) = doSQL($sql);
221
222		return ($rs, '') if ($rs != 0);
223
224		}
225
226		push_el(\@main::el, 'collect_http_traffic()', 'Ending...');
227
228		return (0, $timestamp);
	243	my ($key, $value,$timestamp,$rs) = (undef, undef, time(),undef);
	244
	245	push_el(\@main::el, 'collect_http_traffic()', 'Starting...');
	246
	247	while (($key,$value) = each(%main::domain_name_id)) {
	248	($rs, $timestamp)=collect_http_traffic_from_log($key,$value);
	249	return ($rs, '') if ($rs != 0);
	250	}
	251
	252	while (($key,$value) = each(%main::als_name_id)) {
	253	($rs, $timestamp)=collect_http_traffic_from_log($key,$main::als_id_parentid{$value});
	254	return ($rs, '') if ($rs != 0);
	255	}
	256
	257	while (($key,$value) = each(%main::sub_id_name)) {
	258	($rs, $timestamp)=collect_http_traffic_from_log("$value.$main::domain_id_name{$main::sub_id_parentid{$key}}",$main::sub_id_parentid{$key});
	259	return ($rs, '') if ($rs != 0);
	260	}
	261
	262	push_el(\@main::el, 'collect_http_traffic()', 'Ending...');
	263
	264	return (0, $timestamp);
229	265	}
230	266
231	267	sub collect_ftp_traffic {
232	268
233		my ($timestamp) = @_;
234
235		my ($key, $value, $id) = (undef, undef, undef);
236
237		my ($rs, $rdata, $rlog) = (undef, undef, undef);
	269	my ($timestamp) = @_;
	270
	271	my ($key, $value, $id) = (undef, undef, undef);
	272
	273	my ($rs, $rdata, $rlog) = (undef, undef, undef);
238	274
239	275	my $traffic_size = undef;
240	276	my $traffic_domain = undef;
241	277
242		($rs, $rlog) = gen_log_file($main::cfg{'FTP_TRAFF_LOG'}, $main::cfg{'TRAFF_LOG_DIR'}, $main::cfg{'LOG_DIR'});
243
244		return $rs if ($rs != 0);
245
246		if ($rlog ne '_no_') {
247
248		my $res = open(F, '<', $rlog);
249
250		if (!defined($res)) {
251		push_el(\@main::el, 'collect_ftp_traffic()', "ERROR: Can't open '$rlog' for reading !");
252		return (-1, '');
253		}
254
255		while (<F>) {
256
257		my $line = $_;
258
259		if ($line =~ /^(\d+) [^\@]+\@([^\n]+)\n$/) {
	278	($rs, $rlog) = gen_log_file($main::cfg{'FTP_TRAFF_LOG'}, $main::cfg{'TRAFF_LOG_DIR'}, $main::cfg{'LOG_DIR'});
	279
	280	return $rs if ($rs != 0);
	281
	282	if ($rlog ne '_no_') {
	283
	284	my $res = open(F, '<', $rlog);
	285
	286	if (!defined($res)) {
	287	push_el(\@main::el, 'collect_ftp_traffic()', "ERROR: Can't open '$rlog' for reading !");
	288	return (-1, '');
	289	}
	290
	291	while (<F>) {
	292
	293	my $line = $_;
	294
	295	if ($line =~ /^(\d+) [^\@]+\@([^\n]+)\n$/) {
260	296
261	297	$traffic_size = $1;
…	…
269	305	}
270	306
271		}
272
273		}
274
275		close(F);
276
277		while (($key, $value) = each %main::ftp_traffic) {
278
279		if ($id = $main::als_name_id{$key}) {
280		$main::ftp_traffic{
281		$main::domain_id_name {
282		$main::als_id_parentid{$id}
283		}
284		} += $value;
285		}
286
287		}
288
289		$rs = del_file($rlog);
290
291		return $rs if ($rs != 0);
292		}
293
294		while (($key, $value) = each %main::ftp_traffic) {
295
296		if ($id = $main::domain_name_id{$key}) {
297
298		my $sql = undef;
299
300		$sql = "update domain_traffic set dtraff_ftp = $value where dtraff_time = $timestamp and domain_id = $id;";
301
302		($rs, $rdata) = doSQL($sql);
303
304		return $rs if ($rs != 0);
305
306		}
307		}
	307	}
	308
	309	}
	310
	311	close(F);
	312
	313	while (($key, $value) = each %main::ftp_traffic) {
	314
	315	if ($id = $main::als_name_id{$key}) {
	316	$main::ftp_traffic{
	317	$main::domain_id_name {
	318	$main::als_id_parentid{$id}
	319	}
	320	} += $value;
	321	}
	322
	323	}
	324
	325	$rs = del_file($rlog);
	326
	327	return $rs if ($rs != 0);
	328	}
	329
	330	while (($key, $value) = each %main::ftp_traffic) {
	331
	332	if ($id = $main::domain_name_id{$key}) {
	333
	334	if($value != 0){
	335
	336	my $sql = undef;
	337
	338	$sql="select COUNT(`domain_id`) from domain_traffic where `domain_id`='$id' and `dtraff_time`>=UNIX_TIMESTAMP(CURRENT_DATE())";
	339
	340	my ($rs, $rows) = doSQL($sql);
	341
	342	return $rs if ($rs != 0);
	343
	344	if($$rows[0][0] == 0){
	345
	346	$sql = "insert into `domain_traffic` (domain_id, dtraff_time, dtraff_web, dtraff_ftp, dtraff_mail, dtraff_pop) values ('$id', '$timestamp', '0', $value, '0', '0');";
	347
	348	} else {
	349
	350	$sql = "update `domain_traffic` set `dtraff_ftp`=`dtraff_ftp`+$value, `dtraff_time`='$timestamp', `correction`=0 where `domain_id`='$id' and `dtraff_time`>=UNIX_TIMESTAMP(CURRENT_DATE()) LIMIT 1";
	351
	352	}
	353
	354	($rs, $rdata) = doSQL($sql);
	355
	356	return $rs if ($rs != 0);
	357
	358	}
	359
	360	}
	361	}
308	362	}
309	363
310	364	sub collect_mail_traffic {
311	365
312		my ($timestamp) = @_;
313
314		my ($key, $value, $id) = (undef, undef, undef);
315
316		my ($rs, $rdata, $rlog) = (undef, undef, undef);
317
318		my ($cat, $grep, $awk) = ($main::cfg{'CMD_CAT'}, $main::cfg{'CMD_GREP'}, $main::cfg{'CMD_AWK'});
	366	my ($timestamp) = @_;
	367
	368	my ($key, $value, $id) = (undef, undef, undef);
	369
	370	my ($rs, $rdata, $rlog) = (undef, undef, undef);
	371
	372	my ($cat, $grep, $awk) = ($main::cfg{'CMD_CAT'}, $main::cfg{'CMD_GREP'}, $main::cfg{'CMD_AWK'});
319	373
320	374	my $traffic_size = undef;
321		my $traffic_domain = undef;
	375	my $traffic_domain = undef;
322	376	my $smtp_from_domain = undef;
323	377	my $smtp_to_domain = undef;
324	378
325		($rs, $rlog) = gen_log_file($main::cfg{'MAIL_TRAFF_LOG'}, $main::cfg{'TRAFF_LOG_DIR'}, $main::cfg{'LOG_DIR'});
326
327		return $rs if ($rs != 0);
328
329
330		if ($rlog ne '_no_') {
331
332		my $smtp_log = "$main::cfg{'LOG_DIR'}/smtp.log";
333
334		my $smtp_delivery_log = "$main::cfg{'LOG_DIR'}/smtp_delivery.log";
335
336		my $imap_log = "$main::cfg{'LOG_DIR'}/imap.log";
337
338		my $pop3_log = "$main::cfg{'LOG_DIR'}/pop3.log";
339
340		my $cmd = "$grep 'postfix' $rlog \| $main::cfg{'CMD_PFLOGSUM'} standard 1>$smtp_delivery_log";
341
342		$rs = sys_command($cmd);
343
344		return $rs if ($rs != 0);
345
346		if(exists $main::cfg{'MAIL_LOG_INC_AMAVIS'} && $main::cfg{'MAIL_LOG_INC_AMAVIS'}) {
	379	($rs, $rlog) = gen_log_file($main::cfg{'MAIL_TRAFF_LOG'}, $main::cfg{'TRAFF_LOG_DIR'}, $main::cfg{'LOG_DIR'});
	380
	381	return $rs if ($rs != 0);
	382
	383
	384	if ($rlog ne '_no_') {
	385
	386	my $smtp_log = "$main::cfg{'LOG_DIR'}/smtp.log";
	387
	388	my $smtp_delivery_log = "$main::cfg{'LOG_DIR'}/smtp_delivery.log";
	389
	390	my $imap_log = "$main::cfg{'LOG_DIR'}/imap.log";
	391
	392	my $pop3_log = "$main::cfg{'LOG_DIR'}/pop3.log";
	393
	394	my $cmd = "$grep 'postfix' $rlog \| $main::cfg{'CMD_PFLOGSUM'} standard 1>$smtp_delivery_log";
	395
	396	$rs = sys_command($cmd);
	397
	398	return $rs if ($rs != 0);
	399
	400	if(exists $main::cfg{'MAIL_LOG_INC_AMAVIS'} && $main::cfg{'MAIL_LOG_INC_AMAVIS'}) {
347	401	# Logs including AMAVIS/SPAMASSASSIN/CLAMAV entries
348	402	$cmd = "$cat $smtp_delivery_log \| $awk '{if ( \$10 != \"?\" && !(\$5 ~ /localhost\|127.0.0.1/ && \$6 ~ /localhost\|127.0.0.1/)) print substr(\$3, index(\$3, \"@\")+1, length(\$3)),substr(\$4, index(\$4, \"@\")+1, length(\$4)),\$10;}' 1>$smtp_log";
…	…
352	406	}
353	407
354		$rs = sys_command($cmd);
355
356		return $rs if ($rs != 0);
357
358		$rs = del_file($smtp_delivery_log);
359
360		return $rs if ($rs != 0);
361
362		my $res = open(F, '<', $smtp_log);
363
364		if (!defined($res)) {
365		push_el(\@main::el, 'collect_mail_traffic()', "ERROR: Can't open '$smtp_log' for reading !");
366		return (-1, '');
367		}
368
369		while (<F>) {
370
371		my $line = $_;
372
373		if ($line =~ /^([^\,]+)\ ([^\,]+)\ (\d+)\n$/) {
	408	$rs = sys_command($cmd);
	409
	410	return $rs if ($rs != 0);
	411
	412	# $rs = del_file($smtp_delivery_log);
	413
	414	return $rs if ($rs != 0);
	415
	416	my $res = open(F, '<', $smtp_log);
	417
	418	if (!defined($res)) {
	419	push_el(\@main::el, 'collect_mail_traffic()', "ERROR: Can't open '$smtp_log' for reading !");
	420	return (-1, '');
	421	}
	422
	423	while (<F>) {
	424
	425	my $line = $_;
	426
	427	if ($line =~ /^([^\,]+)\ ([^\,]+)\ (\d+)\n$/) {
374	428
375	429	$smtp_from_domain = $1;
…	…
383	437	$smtp_to_domain = $3.$4;
384	438	}
385
386		if(exists $main::smtp_traffic{$smtp_from_domain}) {
387		$main::smtp_traffic{$smtp_from_domain} += $traffic_size;
	439	if(exists $main::smtp_traffic{$smtp_from_domain}) {
	440	$main::smtp_traffic{$smtp_from_domain} += $traffic_size;
388	441	}
389	442	else {
390		$main::smtp_traffic{$smtp_to_domain} += $traffic_size;
391		}
392
393		}
394
395		}
396
397		close(F);
398
399		while (($key, $value) = each %main::smtp_traffic) {
400
401		if ($id = $main::als_name_id{$key}) {
402		$main::smtp_traffic{
403		$main::domain_id_name {
404		$main::als_id_parentid{$id}
405		}
406		} += $value;
407		}
408
409		}
410
411		$rs = del_file($smtp_log);
412
413		return $rs if ($rs != 0);
	443	$main::smtp_traffic{$smtp_to_domain} += $traffic_size;
	444	}
	445
	446	}
	447
	448	}
	449
	450	close(F);
	451
	452	while (($key, $value) = each %main::smtp_traffic) {
	453
	454	if ($id = $main::als_name_id{$key}) {
	455	$main::smtp_traffic{
	456	$main::domain_id_name {
	457	$main::als_id_parentid{$id}
	458	}
	459	} += $value;
	460	}
	461
	462	}
	463
	464	# $rs = del_file($smtp_log);
	465
	466	return $rs if ($rs != 0);
414	467
415	468
…	…
418	471	$rs = sys_command("$cat $rlog\|$grep 'imapd'\|$grep 'user='\|$grep 'body='\|$grep -v 'ip=\\[::ffff:127.0.0.1\\]'\|$awk '{print \$7,\$9,substr(\$10,1,length(\$10)-1);}' 1>$imap_log");
419	472
420		return $rs if ($rs != 0);
421
422		$res = open(F, '<', $imap_log);
423
424		if (!defined($res)) {
425		push_el(\@main::el, 'collect_mail_traffic()', "ERROR: Can't open '$imap_log' for reading !");
426		return (-1, '');
427		}
428
429		while (<F>) {
430
431		my $line = $_;
432
433		if ($line =~ /^user=([^\,]+)\, headers=(\d+), body=(\d+)\n$/) {
434
435		my ($ma, $sum) = ("$1\n", $2 + $3);
436
437		$ma =~ /^([^\@]+)\@([^\n]+)\n$/;
	473	return $rs if ($rs != 0);
	474
	475	$res = open(F, '<', $imap_log);
	476
	477	if (!defined($res)) {
	478	push_el(\@main::el, 'collect_mail_traffic()', "ERROR: Can't open '$imap_log' for reading !");
	479	return (-1, '');
	480	}
	481
	482	while (<F>) {
	483
	484	my $line = $_;
	485
	486	if ($line =~ /^user=([^\,]+)\, headers=(\d+), body=(\d+)\n$/) {
	487
	488	my ($ma, $sum) = ("$1\n", $2 + $3);
	489
	490	$ma =~ /^([^\@]+)\@([^\n]+)\n$/;
438	491
439	492	$traffic_domain = $2;
…	…
446	499	}
447	500
448		}
449
450		}
451
452		close(F);
453
454
455		$rs = del_file($imap_log);
456
457		return $rs if ($rs != 0);
	501	}
	502
	503	}
	504
	505	close(F);
	506
	507
	508	$rs = del_file($imap_log);
	509
	510	return $rs if ($rs != 0);
458	511
459	512
…	…
462	515	$rs = sys_command("$cat $rlog\|$grep 'imapd-ssl'\|$grep 'user='\|$grep -v 'ip=\\[::ffff:127.0.0.1\\]'\|$awk '{print \$7,\$9,substr(\$10,1,length(\$10)-1);}' 1>$imap_log");
463	516
464		return $rs if ($rs != 0);
465
466		$res = open(F, '<', $imap_log);
467
468		if (!defined($res)) {
469		push_el(\@main::el, 'collect_mail_traffic()', "ERROR: Can't open '$imap_log' for reading !");
470		return (-1, '');
471		}
472
473		while (<F>) {
474
475		my $line = $_;
476
477		if ($line =~ /^user=([^\,]+)\, headers=(\d+), body=(\d+)\n$/) {
478
479		my ($ma, $sum) = ("$1\n", $2 + $3);
480
481		$ma =~ /^([^\@]+)\@([^\n]+)\n$/;
	517	return $rs if ($rs != 0);
	518
	519	$res = open(F, '<', $imap_log);
	520
	521	if (!defined($res)) {