#
# http://www.fabiankeil.de/sourcecode/privoxy-log-parser/
#
-# $Id: privoxy-log-parser.pl,v 1.95 2010/10/23 08:01:36 fabiankeil Exp $
+# $Id: privoxy-log-parser.pl,v 1.125 2011/12/31 14:53:47 fabiankeil Exp $
#
# TODO:
# - LOG_LEVEL_CGI, LOG_LEVEL_ERROR, LOG_LEVEL_WRITE content highlighting
use Getopt::Long;
use constant {
- PRIVOXY_LOG_PARSER_VERSION => '0.6',
+ PRIVOXY_LOG_PARSER_VERSION => '0.7',
# Feel free to mess with these ...
DEFAULT_BACKGROUND => 'black', # Choose registered colour (like 'black')
DEFAULT_TEXT_COLOUR => 'white', # Choose registered colour (like 'black')
CLI_OPTION_SHOW_INEFFECTIVE_FILTERS => 0,
CLI_OPTION_ACCEPT_UNKNOWN_MESSAGES => 0,
CLI_OPTION_STATISTICS => 0,
+ CLI_OPTION_UNBREAK_LINES_ONLY => 0,
CLI_OPTION_URL_STATISTICS_THRESHOLD => 0,
+ CLI_OPTION_HOST_STATISTICS_THRESHOLD => 0,
SUPPRESS_SUCCEEDED_FILTER_ADDITIONS => 1,
SHOW_SCAN_INTRO => 0,
'action-bits-update' => 'light_red',
'configuration-line' => 'red',
'content-type' => 'yellow',
+ 'HOST' => HEADER_DEFAULT_COLOUR,
);
%h_colours = %h;
my $message = shift;
if (LOG_UNPARSED_LINES_TO_EXTRA_FILE) {
- open(ERRORLOG, ">>" . ERROR_LOG_FILE) || die "Writing " . ERROR_LOG_FILE . " failed";
- print ERRORLOG $message;
- close(ERRORLOG);
+ open(my $errorlog_fd, ">>" . ERROR_LOG_FILE) || die "Writing " . ERROR_LOG_FILE . " failed";
+ print $errorlog_fd $message;
+ close($errorlog_fd);
}
}
} elsif ($c =~ m/^(scan: )(\w+ .+ HTTP\/\d\.\d)/) {
- # scan: HTTP/1.1 200 OK
+ # scan: GET http://p.p/ HTTP/1.1
$c = $1 . highlight_request_line($2);
} elsif ($c =~ m/^(scan: )((?:HTTP\/\d\.\d|ICY) (\d+) (.*))/) {
- # Server response line
+ # scan: HTTP/1.1 200 OK
$req{$t}{'response_line'} = $2;
$req{$t}{'status_code'} = $3;
$req{$t}{'status_message'} = $4;
or $c =~ m/^Removing 'Connection: close' to imply keep-alive./
or $c =~ m/^keep-alive support is disabled/
or $c =~ m/^Continue hack in da house/
+ or $c =~ m/^Merged multiple header lines to:/
+ or $c =~ m/^Added header: /
)
{
# XXX: Some of these may need highlighting
# Removing 'Connection: close' to imply keep-alive.
# keep-alive support is disabled. Crunching: Keep-Alive: 300.
# Continue hack in da house.
+ # Merged multiple header lines to: 'X-FORWARDED-PROTO: http X-HOST: 127.0.0.1'
+ # Added header: Content-Encoding: deflate
} elsif ($c =~ m/^scanning headers for:/) {
$c =~ s@(?<= from )(\d+)@$h{'Number'}$1$h{'Standard'}@;
$c =~ s@(?<= to )(\d+)@$h{'Number'}$1$h{'Standard'}@;
+ } elsif ($c =~ m/^Killed all-caps Host header line: HOST:/) {
+
+ # Killed all-caps Host header line: HOST: bestproxydb.com
+ $c = highlight_matched_host($c, '(?<=HOST: )[^\s]+');
+ $c = highlight_matched_pattern($c, 'HOST', 'HOST');
+
} else {
found_unknown_content($c);
# Adding dynamic re_filter job s@^(?:\w*)\s+.*\s+HTTP/\d\.\d\s*@IP-ADDRESS: $origin@D\
# to filter client-ip-address succeeded.
+ } elsif ($c =~ m/^Compressed content from /) {
+
+ # Compressed content from 29258 to 8630 bytes. Compression level: 3
+ $content =~ s@(?<=from )(\d+)@$h{'Number'}$1$h{'Standard'}@;
+ $content =~ s@(?<=to )(\d+)@$h{'Number'}$1$h{'Standard'}@;
+ $content =~ s@(?<=level: )(\d+)@$h{'Number'}$1$h{'Standard'}@;
+
} elsif ($c =~ m/^Reading in filter/) {
return '' unless SHOW_FILTER_READIN_IN;
# XXX: assume the same?
$c = highlight_matched_url($c, '(?<=assuming that \")[^"]*');
+ } elsif ($c =~ m/^Percent-encoding redirect/) {
+
+ # Percent-encoding redirect URL: http://www.example.org/\x02
+ $c = highlight_matched_url($c, '(?<=redirect URL: ).*');
+
} else {
found_unknown_content($c);
} elsif ($content =~ m/^(?:No GIF header found|failed while parsing)/) {
# No GIF header found (XXX: Did I ever commit this?)
- # failed while parsing 195 134747048 (XXX: never commited)
+ # failed while parsing 195 134747048 (XXX: never committed)
# Ignore these for now
} elsif ($c =~ m/^Waiting for the next client request/ or
$c =~ m/^The connection on server socket/ or
- $c =~ m/^Client request arrived in time or the client closed the connection/) {
+ $c =~ m/^Client request arrived in time /) {
# Waiting for the next client request on socket 3. Keeping the server \
# socket 12 to a.fsdn.com open.
# The connection on server socket 6 to upload.wikimedia.org isn't reusable. Closing.
+ # Used by Privoxy 3.0.18 and later:
+ # Client request arrived in time on socket 21.
+ # Used by earlier version:
# Client request arrived in time or the client closed the connection on socket 12.
$c =~ s@(?<=on socket )(\d+)@$h{'Number'}$1$h{'Standard'}@;
$c =~ s@(?<=set to )(\d+)@$h{'Number'}$1$h{'Standard'}@;
$c =~ s@(?<=reading )(\d+)@$h{'Number'}$1$h{'Standard'}@;
+ } elsif ($c =~ m/^Reducing expected bytes to /) {
+
+ # Reducing expected bytes to 0. Marking the server socket tainted after throwing 4 bytes away.
+ $c =~ s@(?<=bytes to )(\d+)@$h{'Number'}$1$h{'Standard'}@;
+ $c =~ s@(?<=after throwing )(\d+)@$h{'Number'}$1$h{'Standard'}@;
+
} elsif ($c =~ m/^Waiting for up to /) {
# Waiting for up to 4999 bytes from the client.
} elsif ($c =~ m/^Decompress(ing deflated|ion didn)/ or
$c =~ m/^Compressed content detected/ or
+ $c =~ m/^SDCH-compressed content detected/ or
$c =~ m/^Tagger/
) {
# Decompressing deflated iob: 117
# Decompression didn't result in any content.
# Compressed content detected, content filtering disabled. Consider recompiling Privoxy\
# with zlib support or enable the prevent-compression action.
+ # SDCH-compressed content detected, content filtering disabled.\
+ # Consider suppressing SDCH offers made by the client.
# Tagger 'complete-url' created empty tag. Ignored.
# Ignored for now
# Request from 10.0.0.1 denied. limit-connect{,} doesn't allow CONNECT requests to port 443.
# Request from 10.0.0.1 marked for blocking. limit-connect{,} doesn't allow CONNECT requests to port 443.
+ # 3.0.18 and later:
+ # Request from 10.0.0.1 marked for blocking. limit-connect{0} doesn't allow CONNECT requests to www.example.org:443
# Malformed server response detected. Downgrading to HTTP/1.0 impossible.
$c =~ s@(?<=Request from )([^\s]*)@$h{'ip-address'}$1$h{'Standard'}@;
$c =~ s@(denied|blocking)@$h{'warning'}$1$h{'Standard'}@;
$c =~ s@(CONNECT)@$h{'method'}$1$h{'Standard'}@;
$c =~ s@(?<=to port )(\d+)@$h{'port'}$1$h{'Standard'}@;
+ $c =~ s@(?<=to )([^\s]+)@$h{'request_'}$1$h{'Standard'}@;
} elsif ($c =~ m/^Status code/) {
$c =~ s@(?<=Needed: )(\d+)@$h{'Number'}$1$h{'Standard'}@;
$c =~ s@(?<=Limit: )(\d+)@$h{'Number'}$1$h{'Standard'}@;
+ } elsif ($c =~ m/^File modification detected: /) {
+
+ # File modification detected: /usr/local/etc/privoxy/user-agent.action
+ $c =~ s@(?<= detected: )(.*)$@$h{'file'}$1$h{'Standard'}@;
+
} elsif ($c =~ m/^No logfile configured/ or
$c =~ m/^Malformerd HTTP headers detected and MS IIS5 hack enabled/ or
$c =~ m/^Invalid \"chunked\" transfer/ or
my $c = shift;
- if ($c =~ m/^Empty server or forwarder response received on socket \d+./) {
+ if ($c =~ m/^(?:Empty|No) server or forwarder response received on socket \d+\./) {
# Empty server or forwarder response received on socket 4.
# Empty server or forwarder response received on socket 3. \
# Closing client socket 15 without sending data.
+ # Used by Privoxy 3.0.18 and later:
+ # No server or forwarder response received on socket 8. \
+ # Closing client socket 10 without sending data.
+
$c =~ s@(?<=on socket )(\d+)@$h{'Number'}$1$h{'Standard'}@;
$c =~ s@(?<=client socket )(\d+)@$h{'Number'}$1$h{'Standard'}@;
+
+ } elsif ($c =~ m/^Didn't receive data in time:/) {
+
+ # Didn't receive data in time: a.fsdn.com:443
+ $c =~ s@(?<=in time: )(.*)@$h{'destination'}$1$h{'Standard'}@;
}
+
# XXX: There are probably more messages that deserve highlighting.
return $c;
# scan: HTTP/1.1 200 OK
$stats{'method'}{$2}++;
- $stats{'ressource'}{$3}++;
+ $stats{'resource'}{$3}++;
$stats{'http-version'}{$4}++;
+
+ } elsif ($c =~ m/^scan: Host: ([^\s]+)/) {
+
+ # scan: Host: p.p
+ $stats{'hosts'}{$1}++;
}
}
'empty-responses' => 0,
'empty-responses-on-new-connections' => 0,
'empty-responses-on-reused-connections' => 0,
+ 'fast-redirections' => 0,
+ 'blocked' => 0,
+ 'reused-connections' => 0,
+ 'server-keep-alive' => 0,
);
}
print "URL statistics are disabled. Increase --url-statistics-threshold to enable them.\n";
} else {
print "Requested URLs:\n";
- foreach my $ressource (sort {$stats{'ressource'}{$b} <=> $stats{'ressource'}{$a}} keys %{$stats{'ressource'}}) {
- if ($stats{'ressource'}{$ressource} < $cli_options{'url-statistics-threshold'}) {
+ foreach my $resource (sort {$stats{'resource'}{$b} <=> $stats{'resource'}{$a}} keys %{$stats{'resource'}}) {
+ if ($stats{'resource'}{$resource} < $cli_options{'url-statistics-threshold'}) {
print "Skipped statistics for URLs below the treshold.\n";
last;
}
- printf "%d : %s\n", $stats{'ressource'}{$ressource}, $ressource;
+ printf "%d : %s\n", $stats{'resource'}{$resource}, $resource;
+ }
+ }
+
+ if ($cli_options{'host-statistics-threshold'} == 0) {
+ print "Host statistics are disabled. Increase --host-statistics-threshold to enable them.\n";
+ } else {
+ print "Requested Hosts:\n";
+ foreach my $host (sort {$stats{'hosts'}{$b} <=> $stats{'hosts'}{$a}} keys %{$stats{'hosts'}}) {
+ if ($stats{'hosts'}{$host} < $cli_options{'host-statistics-threshold'}) {
+ print "Skipped statistics for Hosts below the treshold.\n";
+ last;
+ }
+ printf "%d : %s\n", $stats{'hosts'}{$host}, $host;
}
}
}
while (<>) {
- if (m/^(\w{3} \d{2}) (\d\d:\d\d:\d\d)\.?(\d+)? (?:Privoxy\()?([^\)\s]*)[\)]? ([\w -]*): (.*?)\r?$/) {
+ if (m/^(\d{4}-\d{2}-\d{2}|\w{3} \d{2}) (\d\d:\d\d:\d\d)\.?(\d+)? (?:Privoxy\()?([^\)\s]*)[\)]? ([\w -]*): (.*?)\r?$/) {
$thread = $t = ($shorten_thread_ids) ? shorten_thread_id($4) : $4;
$req{$t}{'day'} = $day = $1;
$req{$t}{'time-stamp'} = $time_stamp = $2;
'Error' => \&gather_loglevel_error_stats,
'Fatal error' => \&handle_loglevel_ignore,
'Writing' => \&handle_loglevel_ignore,
+ 'Received' => \&handle_loglevel_ignore,
'Unknown log level' => \&handle_loglevel_ignore
);
while (<>) {
- if (m/^(\w{3} \d{2}) (\d\d:\d\d:\d\d)\.?(\d+)? (?:Privoxy\()?([^\)\s]*)[\)]? ([\w -]*): (.*?)\r?$/) {
+ if (m/^(\d{4}-\d{2}-\d{2}|\w{3} \d{2}) (\d\d:\d\d:\d\d)\.?(\d+)? (?:Privoxy\()?([^\)\s]*)[\)]? ([\w -]*): (.*?)\r?$/) {
$day = $1;
$time_stamp = $2;
$msecs = $3 ? $3 : 0;
}
+sub unbreak_lines_only_loop() {
+ my $log_messages_reached = 0;
+ while (<>) {
+ chomp;
+
+ # Log level other than LOG_LEVEL_CLF?
+ if (m/^(\d{4}-\d{2}-\d{2}|\w{3} \d{2}) (\d\d:\d\d:\d\d)\.?(\d+)? (?:Privoxy\()?([^\)\s]*)[\)]? ([\w -]*): (.*?)\r?$/ or
+ # LOG_LEVEL_CLF?
+ m/^((?:\d+\.\d+\.\d+\.\d+)) - - \[(.*)\] "(.*)" (\d+) (\d+)/) {
+ $log_messages_reached = 1;
+ print "\n";
+
+ } else {
+ # Wrapped message
+ $_ = "\n". $_ if /^(?:\d+\.\d+\.\d+\.\d+)/;
+ $_ = " " . $_;
+ }
+ s@<BR>$@@;
+ print;
+ print "\n" unless $log_messages_reached;
+ }
+ print "\n";
+}
+
sub VersionMessage {
my $version_message;
'show-ineffective-filters' => CLI_OPTION_SHOW_INEFFECTIVE_FILTERS,
'accept-unknown-messages' => CLI_OPTION_ACCEPT_UNKNOWN_MESSAGES,
'statistics' => CLI_OPTION_STATISTICS,
- 'url-statistics-threshold' => CLI_OPTION_URL_STATISTICS_THRESHOLD,
+ 'url-statistics-threshold' => CLI_OPTION_URL_STATISTICS_THRESHOLD,
+ 'unbreak-lines-only' => CLI_OPTION_UNBREAK_LINES_ONLY,
+ 'host-statistics-threshold'=> CLI_OPTION_HOST_STATISTICS_THRESHOLD,
);
GetOptions (
'show-ineffective-filters' => \$cli_options{'show-ineffective-filters'},
'accept-unknown-messages' => \$cli_options{'accept-unknown-messages'},
'statistics' => \$cli_options{'statistics'},
- 'url-statistics-threshold=s'=> \$cli_options{'url-statistics-threshold'},
+ 'unbreak-lines-only' => \$cli_options{'unbreak-lines-only'},
+ 'url-statistics-threshold=i'=> \$cli_options{'url-statistics-threshold'},
+ 'host-statistics-threshold=i'=> \$cli_options{'host-statistics-threshold'},
'version' => sub { VersionMessage && exit(0) },
'help' => \&help,
) or exit(1);
Options and their default values if they have any:
[--accept-unknown-messages]
+ [--host-statistics-threshold $cli_options{'host-statistics-threshold'}]
[--html-output]
[--no-embedded-css]
[--no-msecs]
[--shorten-thread-ids]
[--show-ineffective-filters]
[--statistics]
+ [--unbreak-lines-only]
[--url-statistics-threshold $cli_options{'url-statistics-threshold'}]
[--title $cli_options{'title'}]
[--version]
print_intro();
- if (cli_option_is_set('statistics')) {
+ # XXX: should explicitly reject incompatible argument combinations
+ if (cli_option_is_set('unbreak-lines-only')) {
+ unbreak_lines_only_loop();
+ } elsif (cli_option_is_set('statistics')) {
stats_loop();
} else {
parse_loop();
[B<--accept-unknown-messages>] Don't print warnings in case of unknown messages,
just don't highlight them.
+[B<--host-statistics-threshold>] Only show the request count for a host
+if it's above or equal to the given threshold. If the threshold is 0, host
+statistics are disabled.
+
[B<--html-output>] Use HTML and CSS for the syntax highlighting. If this option is
omitted, ANSI escape sequences are used unless B<--no-syntax-highlighting> is active.
This option is only intended to make embedding log excerpts in web pages easier.
they very well might be. Also note that the results are pretty much guaranteed
to be incorrect if Privoxy and Privoxy-Log-Parser aren't in sync.
-[B<--url-statistics-threshold>] Only show the request count for a ressource
+[B<--unbreak-lines-only>] Tries to fix lines that got messed up by a broken or
+interestingly configured mail client and thus are no longer recognized properly.
+Only fixes some breakage, but may be good enough or at least better than nothing.
+Doesn't do anything else, so you probably want to pipe the output into
+B<privoxy-log-parser> again.
+
+[B<--url-statistics-threshold>] Only show the request count for a resource
if it's above or equal to the given threshold. If the threshold is 0, URL
statistics are disabled.