#
# https://www.fabiankeil.de/sourcecode/privoxy-log-parser/
#
-# $Id: privoxy-log-parser.pl,v 1.167 2017/02/24 12:00:16 fabiankeil Exp $
-#
# TODO:
# - LOG_LEVEL_CGI, LOG_LEVEL_ERROR, LOG_LEVEL_WRITE content highlighting
# - create fancy statistics
# hash key as input.
# - Add --compress and --decompress options.
#
-# Copyright (c) 2007-2017 Fabian Keil <fk@fabiankeil.de>
+# Copyright (c) 2007-2020 Fabian Keil <fk@fabiankeil.de>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
use Getopt::Long;
use constant {
- PRIVOXY_LOG_PARSER_VERSION => '0.9',
+ PRIVOXY_LOG_PARSER_VERSION => '0.9.1',
# Feel free to mess with these ...
DEFAULT_BACKGROUND => 'black', # Choose registered colour (like 'black')
DEFAULT_TEXT_COLOUR => 'white', # Choose registered colour (like 'black')
my $shorten_thread_ids;
my $line_end;
-sub prepare_our_stuff () {
+sub prepare_our_stuff() {
# Syntax Higlight hash
@all_colours = (
init_stats();
}
-sub paint_it ($) {
+sub paint_it($) {
###############################################################
# Takes a colour string and returns an ANSI escape sequence
# (unless --no-syntax-highlighting is used).
return $colour_code;
}
-sub get_semantic_html_markup ($) {
+sub get_semantic_html_markup($) {
###############################################################
# Takes a string and returns a span element
###############################################################
return $code;
}
-sub cli_option_is_set ($) {
+sub cli_option_is_set($) {
our %cli_options;
my $cli_option = shift;
return $cli_options{$cli_option};
}
-sub get_html_title () {
+sub get_html_title() {
our %cli_options;
return $cli_options{'title'};
);
}
-sub get_css_colour ($) {
+sub get_css_colour($) {
our %css_colours;
my $colour = shift;
return '#' . $css_colours{$colour};
}
-sub get_css_line ($) {
+sub get_css_line($) {
my $class = shift;
my $css_line;
return $css_line;
}
-sub get_css_line_for_colour ($) {
+sub get_css_line_for_colour($) {
my $colour = shift;
my $css_line;
}
# XXX: Wrong solution
-sub get_missing_css_lines () {
+sub get_missing_css_lines() {
my $css_line;
return $css_line;
}
-sub get_css () {
+sub get_css() {
our %css_colours; #XXX: Wrong solution
return $css;
}
-sub print_intro () {
+sub print_intro() {
my $intro = '';
}
}
-sub print_outro () {
+sub print_outro() {
my $outro = '';
}
}
-sub get_line_end () {
+sub get_line_end() {
return cli_option_is_set('html-output') ? "<br>\n" : "\n";
}
-sub get_colour_html_markup ($) {
+sub get_colour_html_markup($) {
###############################################################
# Takes a colour string a span element. XXX: WHAT?
# XXX: This function shouldn't be necessary, the
return $code;
}
-sub default_colours () {
+sub default_colours() {
# XXX: Properly
our $bg_code;
return reset_colours();
}
-sub show_colours () {
+sub show_colours() {
# XXX: Implement
}
-sub reset_colours () {
+sub reset_colours() {
return ESCAPE . "0m";
}
-sub set_background ($){
+sub set_background($) {
my $colour = shift;
our $bg_code;
}
}
-sub get_background (){
+sub get_background() {
return our $bg_code;
}
-sub prepare_highlight_hash ($) {
+sub prepare_highlight_hash($) {
my $ref = shift;
foreach my $key (keys %$ref) {
}
}
-sub prepare_colour_array ($) {
+sub prepare_colour_array($) {
my $ref = shift;
foreach my $i (0 ... @$ref - 1) {
}
}
-sub found_unknown_content ($) {
+sub found_unknown_content($) {
my $unknown = shift;
my $message;
die "Unworthy content parser" if PUNISH_MISSING_LOG_KNOWLEDGE_WITH_DEATH;
}
-sub log_parse_error ($) {
+sub log_parse_error($) {
my $message = shift;
}
}
-sub debug_message (@) {
+sub debug_message(@) {
my @message = @_;
print $h{'debug'} . "@message" . $h{'Standard'} . "\n";
# highlighter functions that aren't loglevel-specific
################################################################################
-sub h ($) {
+sub h($) {
# Get highlight marker
my $highlight = shift; # XXX: Stupid name;
return $result;
}
-sub highlight_known_headers ($) {
+sub highlight_known_headers($) {
my $content = shift;
return $content;
}
-sub highlight_matched_request_line ($$) {
+sub highlight_matched_request_line($$) {
my $result = shift; # XXX: Stupid name;
my $regex = shift;
return $result;
}
-sub highlight_request_line ($) {
+sub highlight_request_line($) {
my $rl = shift;
my ($method, $url, $http_version);
return $rl;
}
-sub highlight_response_line ($) {
+sub highlight_response_line($) {
my $rl = shift;
my ($http_version, $status_code, $status_message);
return $rl;
}
-sub highlight_matched_url ($$) {
+sub highlight_matched_url($$) {
my $result = shift; # XXX: Stupid name;
my $regex = shift;
return $result;
}
-sub highlight_matched_host ($$) {
+sub highlight_matched_host($$) {
my ($result, $regex) = @_; # XXX: result ist stupid name;
return $result;
}
-sub highlight_matched_pattern ($$$) {
+sub highlight_matched_pattern($$$) {
my $result = shift; # XXX: Stupid name;
my $key = shift;
return $result;
}
-sub highlight_matched_path ($$) {
+sub highlight_matched_path($$) {
my $result = shift; # XXX: Stupid name;
my $regex = shift;
return $result;
}
-sub highlight_url ($) {
+sub highlight_url($) {
my $url = shift;
return $url;
}
-sub update_header_highlight_regex ($) {
+sub update_header_highlight_regex($) {
my $header = shift;
my $headers = join ('|', keys %header_colours);
# loglevel-specific highlighter functions
################################################################################
-sub handle_loglevel_header ($) {
+sub handle_loglevel_header($) {
my $c = shift;
return $c;
}
-sub handle_loglevel_re_filter ($) {
+sub handle_loglevel_re_filter($) {
my $content = shift;
my $c = $content;
return $content;
}
-sub handle_loglevel_redirect ($) {
+sub handle_loglevel_redirect($) {
my $c = shift;
return $c;
}
-sub handle_loglevel_gif_deanimate ($) {
+sub handle_loglevel_gif_deanimate($) {
my $content = shift;
return $content;
}
-sub handle_loglevel_request ($) {
+sub handle_loglevel_request($) {
my $content = shift;
return $content;
}
-sub handle_loglevel_crunch ($) {
+sub handle_loglevel_crunch($) {
my $content = shift;
return $content;
}
-sub handle_loglevel_connect ($) {
+sub handle_loglevel_connect($) {
my $c = shift;
}
-sub handle_loglevel_info ($) {
+sub handle_loglevel_info($) {
my $c = shift;
return $c;
}
-sub handle_loglevel_cgi ($) {
+sub handle_loglevel_cgi($) {
my $c = shift;
return $c;
}
-sub handle_loglevel_force ($) {
+sub handle_loglevel_force($) {
my $c = shift;
return $c;
}
-sub handle_loglevel_error ($) {
+sub handle_loglevel_error($) {
my $c = shift;
}
-sub handle_loglevel_ignore ($) {
+sub handle_loglevel_ignore($) {
return shift;
}
-sub gather_loglevel_request_stats ($$) {
+sub gather_loglevel_clf_stats($) {
+
+ my $content = shift;
+ my ($method, $resource, $http_version, $status_code, $size);
+ our %stats;
+ our %cli_options;
+
+ # +0200] "GET https://www.youtube.com/watch?v=JmcA9LIIXWw HTTP/1.1" 200 68004
+ $content =~ m/^[+-]\d{4}\] "(\w+) (.+) (HTTP\/\d\.\d)" (\d+) (\d+)/;
+ $method = $1;
+ $resource = $2;
+ $http_version = $3;
+ $status_code = $4;
+ $size = $5;
+
+ unless (defined $method) {
+ print("Failed to parse: $content\n");
+ return;
+ }
+ $stats{'method'}{$method}++;
+ if ($cli_options{'url-statistics-threshold'} != 0) {
+ $stats{'resource'}{$resource}++;
+ }
+ $stats{'http-version'}{$http_version}++;
+
+ if ($cli_options{'host-statistics-threshold'} != 0) {
+ $resource =~ m@(?:http[s]://)([^/]+)/?@;
+ $stats{'hosts'}{$1}++;
+ }
+ $stats{'content-size-total'} += $size;
+ $stats{'status-code'}{$status_code}++;
+ $stats{requests_clf}++;
+}
+
+sub gather_loglevel_request_stats($$) {
my $c = shift;
my $thread = shift;
our %stats;
$stats{requests}++;
}
-sub gather_loglevel_crunch_stats ($$) {
+sub gather_loglevel_crunch_stats($$) {
my $c = shift;
my $thread = shift;
our %stats;
- $stats{requests}++;
$stats{crunches}++;
if ($c =~ m/^Redirected:/) {
}
-sub gather_loglevel_error_stats ($$) {
+sub gather_loglevel_error_stats($$) {
my $c = shift;
my $thread = shift;
}
}
-sub gather_loglevel_connect_stats ($$) {
+sub gather_loglevel_connect_stats($$) {
my ($c, $thread) = @_;
our %thread_data;
}
}
-sub gather_loglevel_header_stats ($$) {
+sub gather_loglevel_header_stats($$) {
my ($c, $thread) = @_;
our %stats;
+ our %cli_options;
if ($c =~ m/^A HTTP\/1\.1 response without/ or
$c =~ m/^Keeping the server header 'Connection: keep-alive' around./)
# A HTTP/1.1 response without Connection header implies keep-alive.
# Keeping the server header 'Connection: keep-alive' around.
$stats{'server-keep-alive'}++;
-
- } elsif ($c =~ m/^scan: ((\w+) (.+) (HTTP\/\d\.\d))/) {
-
- # scan: HTTP/1.1 200 OK
- $stats{'method'}{$2}++;
- $stats{'resource'}{$3}++;
- $stats{'http-version'}{$4}++;
-
- } elsif ($c =~ m/^scan: Host: ([^\s]+)/) {
-
- # scan: Host: p.p
- $stats{'hosts'}{$1}++;
}
}
-sub init_stats () {
+sub init_stats() {
our %stats = (
requests => 0,
+ requests_clf => 0,
crunches => 0,
'server-keep-alive' => 0,
'reused-connections' => 0,
'empty-responses-on-reused-connections' => 0,
'fast-redirections' => 0,
'blocked' => 0,
+ 'connection-failure' => 0,
+ 'connection-timeout' => 0,
'reused-connections' => 0,
'server-keep-alive' => 0,
'closed-client-connections' => 0,
+ 'content-size-total' => 0,
);
$stats{'client-requests-on-connection'}{1} = 0;
}
-sub get_percentage ($$) {
+sub get_percentage($$) {
my $big = shift;
my $small = shift;
return sprintf("%.2f%%", $small / $big * 100);
}
-sub print_stats () {
+sub print_stats() {
our %stats;
our %cli_options;
my $new_connections = $stats{requests} - $stats{crunches} - $stats{'reused-connections'};
- my $outgoing_requests = $stats{requests} - $stats{crunches};
my $client_requests_checksum = 0;
+ if ($stats{requests_clf} && $stats{requests}
+ && $stats{requests_clf} != $stats{requests}) {
+ print "Inconsistent request counts: " . $stats{requests} . "/" . $stats{requests_clf} . "\n";
+ }
+ if ($stats{requests_clf} && $stats{requests} eq 0) {
+ $stats{requests} = $stats{requests_clf};
+ }
+
if ($stats{requests} eq 0) {
print "No requests yet.\n";
return;
}
print "Client requests total: " . $stats{requests} . "\n";
- print "Crunches: " . $stats{crunches} . " (" .
- get_percentage($stats{requests}, $stats{crunches}) . ")\n";
- print "Blocks: " . $stats{'blocked'} . " (" .
- get_percentage($stats{requests}, $stats{'blocked'}) . ")\n";
- print "Fast redirections: " . $stats{'fast-redirections'} . " (" .
- get_percentage($stats{requests}, $stats{'fast-redirections'}) . ")\n";
- print "Connection timeouts: " . $stats{'connection-timeout'} . " (" .
- get_percentage($stats{requests}, $stats{'connection-timeout'}) . ")\n";
- print "Connection failures: " . $stats{'connection-failure'} . " (" .
- get_percentage($stats{requests}, $stats{'connection-failure'}) . ")\n";
- print "Outgoing requests: " . $outgoing_requests . " (" .
- get_percentage($stats{requests}, $outgoing_requests) . ")\n";
+ if ($stats{crunches}) {
+ my $outgoing_requests = $stats{requests} - $stats{crunches};
+ print "Crunches: " . $stats{crunches} . " (" .
+ get_percentage($stats{requests}, $stats{crunches}) . ")\n";
+ print "Blocks: " . $stats{'blocked'} . " (" .
+ get_percentage($stats{requests}, $stats{'blocked'}) . ")\n";
+ print "Fast redirections: " . $stats{'fast-redirections'} . " (" .
+ get_percentage($stats{requests}, $stats{'fast-redirections'}) . ")\n";
+ print "Connection timeouts: " . $stats{'connection-timeout'} . " (" .
+ get_percentage($stats{requests}, $stats{'connection-timeout'}) . ")\n";
+ print "Connection failures: " . $stats{'connection-failure'} . " (" .
+ get_percentage($stats{requests}, $stats{'connection-failure'}) . ")\n";
+ print "Outgoing requests: " . $outgoing_requests . " (" .
+ get_percentage($stats{requests}, $outgoing_requests) . ")\n";
+ } else {
+ print "No crunches detected. Is 'debug 1024' enabled?\n";
+ }
+
print "Server keep-alive offers: " . $stats{'server-keep-alive'} . " (" .
get_percentage($stats{requests}, $stats{'server-keep-alive'}) . ")\n";
print "New outgoing connections: " . $new_connections . " (" .
get_percentage($stats{requests}, $stats{'empty-responses-on-reused-connections'}) .
")\n";
print "Client connections: " . $stats{'closed-client-connections'} . "\n";
-
+ if ($stats{'content-size-total'}) {
+ print "Bytes transfered excluding headers: " . $stats{'content-size-total'} . "\n";
+ }
my $lines_printed = 0;
print "Client requests per connection distribution:\n";
foreach my $client_requests (sort {
printf "%8d : %-8s\n", $stats{'method'}{$method}, $method;
}
} else {
- print "Method distribution unknown. No response headers parsed yet. Is 'debug 8' enabled?\n";
+ print "Method distribution unknown. No CLF message parsed yet. Is 'debug 512' enabled?\n";
}
print "Client HTTP versions:\n";
foreach my $http_version (sort {$stats{'http-version'}{$b} <=> $stats{'http-version'}{$a}} keys %{$stats{'http-version'}}) {
- printf "%d : %s\n", $stats{'http-version'}{$http_version}, $http_version;
+ printf "%8d : %-8s\n", $stats{'http-version'}{$http_version}, $http_version;
+ }
+ if (exists $stats{'status-code'}) {
+ print "HTTP status codes:\n";
+ foreach my $status_code (sort {$stats{'status-code'}{$b} <=> $stats{'status-code'}{$a}} keys %{$stats{'status-code'}}) {
+ printf "%8d : %-8d\n", $stats{'status-code'}{$status_code}, $status_code;
+ }
+ } else {
+ print "Status code distribution unknown. No CLF message parsed yet. Is 'debug 512' enabled?\n";
}
if ($cli_options{'url-statistics-threshold'} == 0) {
# Functions that actually print stuff
################################################################################
-sub print_clf_message () {
+sub print_clf_message() {
our ($ip, $timestamp, $request_line, $status_code, $size);
my $output = '';
print $output;
}
-sub print_non_clf_message ($) {
+sub print_non_clf_message($) {
my $content = shift;
my $msec_string = $no_msecs_mode ? '' : '.' . $req{$t}{'msecs'};
. $line_end;
}
-sub shorten_thread_id ($) {
+sub shorten_thread_id($) {
my $thread_id = shift;
return $short_thread_ids{$thread_id}
}
-sub parse_loop () {
+sub parse_loop() {
my ($day, $time_stamp, $thread, $log_level, $content, $c, $msecs);
my $last_msecs = 0;
}
}
-sub stats_loop () {
+sub stats_loop() {
my ($day, $time_stamp, $msecs, $thread, $log_level, $content);
my $strict_checks = cli_option_is_set('strict-checks');
while (<>) {
(undef, $time_stamp, $thread, $log_level, $content) = split(/ /, $_, 5);
- # Skip LOG_LEVEL_CLF
- next if (not defined($log_level) or $time_stamp eq "-");
- if (defined($log_level_handlers{$log_level})) {
+ next if (not defined($log_level));
+
+ if ($time_stamp eq "-") {
+
+ gather_loglevel_clf_stats($content);
+
+ } elsif (defined($log_level_handlers{$log_level})) {
$content = $log_level_handlers{$log_level}($content, $thread);
print $version_message;
}
-sub get_cli_options () {
+sub get_cli_options() {
our %cli_options = (
'html-output' => CLI_OPTION_DEFAULT_TO_HTML_OUTPUT,
$line_end = get_line_end();
}
-sub help () {
+sub help() {
our %cli_options;
################################################################################
# main
################################################################################
-sub main () {
+sub main() {
get_cli_options();
set_background(DEFAULT_BACKGROUND);
projects / privoxy.git / blobdiff