X-Git-Url: http://www.privoxy.org/gitweb/?p=privoxy.git;a=blobdiff_plain;f=tools%2Fprivoxy-log-parser.pl;h=840360247f49f3426eac3f960a71d8f0b8c6b8bc;hp=2899338d46db301f28d28a2c3abccf84214ee6de;hb=0ef8c061c66fc3847e150434842c2a0552d73097;hpb=dd14a1de75e6f54bdbdbb617e51dd1c04cd9f6bd diff --git a/tools/privoxy-log-parser.pl b/tools/privoxy-log-parser.pl index 2899338d..58d7a556 100755 --- a/tools/privoxy-log-parser.pl +++ b/tools/privoxy-log-parser.pl @@ -6,9 +6,7 @@ # A parser for Privoxy log messages. For incomplete documentation run # perldoc privoxy-log-parser(.pl), for fancy screenshots see: # -# http://www.fabiankeil.de/sourcecode/privoxy-log-parser/ -# -# $Id: privoxy-log-parser.pl,v 1.57 2009/10/10 05:50:02 fabiankeil Exp $ +# https://www.fabiankeil.de/sourcecode/privoxy-log-parser/ # # TODO: # - LOG_LEVEL_CGI, LOG_LEVEL_ERROR, LOG_LEVEL_WRITE content highlighting @@ -23,8 +21,9 @@ # - Handle incomplete input without Perl warning about undefined variables. # - Use generic highlighting function that takes a regex and the # hash key as input. +# - Add --compress and --decompress options. # -# Copyright (c) 2007-2009 Fabian Keil +# Copyright (c) 2007-2021 Fabian Keil # # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above @@ -44,7 +43,7 @@ use warnings; use Getopt::Long; use constant { - PRIVOXY_LOG_PARSER_VERSION => '0.5', + PRIVOXY_LOG_PARSER_VERSION => '0.9.2', # Feel free to mess with these ... DEFAULT_BACKGROUND => 'black', # Choose registered colour (like 'black') DEFAULT_TEXT_COLOUR => 'white', # Choose registered colour (like 'black') @@ -53,20 +52,24 @@ use constant { CLI_OPTION_DEFAULT_TO_HTML_OUTPUT => 0, CLI_OPTION_TITLE => 'Privoxy-Log-Parser in da house', + CLI_OPTION_KEEP_DATE => 0, CLI_OPTION_NO_EMBEDDED_CSS => 0, CLI_OPTION_NO_MSECS => 0, CLI_OPTION_NO_SYNTAX_HIGHLIGHTING => 0, - CLI_OPTION_ERROR_LOG_FILE => '/var/log/privoxy-log.log', + CLI_OPTION_SHORTEN_THREAD_IDS => 0, CLI_OPTION_SHOW_INEFFECTIVE_FILTERS => 0, - CLI_OPTION_ACCEPT_UNKNOWN_MESSAGES => 0, CLI_OPTION_STATISTICS => 0, + CLI_OPTION_STRICT_CHECKS => 0, + CLI_OPTION_UNBREAK_LINES_ONLY => 0, + CLI_OPTION_URL_STATISTICS_THRESHOLD => 0, + CLI_OPTION_HOST_STATISTICS_THRESHOLD => 0, + CLI_OPTION_SHOW_COMPLETE_REQUEST_DISTRIBUTION => 0, SUPPRESS_SUCCEEDED_FILTER_ADDITIONS => 1, SHOW_SCAN_INTRO => 0, SHOW_FILTER_READIN_IN => 0, SUPPRESS_EMPTY_LINES => 1, SUPPRESS_SUCCESSFUL_CONNECTIONS => 1, - SUPPRESS_ACCEPTED_CONNECTIONS => 1, SUPPRESS_GIF_NOT_CHANGED => 1, SUPPRESS_NEED_TO_DE_CHUNK_FIRST => 1, @@ -80,22 +83,46 @@ use constant { PUNISH_MISSING_HIGHLIGHT_KNOWLEDGE_WITH_DEATH => 1, LOG_UNPARSED_LINES_TO_EXTRA_FILE => 0, + ERROR_LOG_FILE => '/var/log/privoxy-log-parser', # You better leave these alone unless you know what you're doing. COLOUR_RESET => "\033[0;0m", ESCAPE => "\033[", }; -sub prepare_our_stuff () { +# For performance reasons, these are global. + +my $t; +my %req; # request data from previous lines +my %h; +my %thread_colours; +my @all_colours; +my @time_colours; +my $thread_colour_index = 0; +my $header_colour_index = 0; +my $time_colour_index = 0; +my %header_colours; +my $no_special_header_highlighting; +my %reason_colours; +my %h_colours; +my $header_highlight_regex = ''; + +my $html_output_mode; +my $keep_date_mode; +my $no_msecs_mode; # XXX: should probably be removed +my $shorten_thread_ids; +my $line_end; + +sub prepare_our_stuff() { # Syntax Higlight hash - our @all_colours = ( + @all_colours = ( 'red', 'green', 'brown', 'blue', 'purple', 'cyan', 'light_gray', 'light_red', 'light_green', 'yellow', 'light_blue', 'pink', 'light_cyan', 'white' ); - our %h = ( + %h = ( # LOG_LEVEL Info => 'blue', Header => 'green', @@ -103,6 +130,7 @@ sub prepare_our_stuff () { 'Re-Filter' => 'purple', Connect => 'brown', Request => 'light_cyan', + Tagging => 'purple', CGI => 'light_green', Redirect => 'cyan', Error => 'light_red', @@ -111,6 +139,8 @@ sub prepare_our_stuff () { 'Gif-Deanimate' => 'blue', Force => 'red', Writing => 'light_green', + Received => 'yellow', + Actions => 'yellow', # ---------------------- URL => 'yellow', path => 'brown', @@ -149,14 +179,16 @@ sub prepare_our_stuff () { 'pcrs-delimiter' => 'light_red', 'ignored' => 'light_red', 'action-bits-update' => 'light_red', + 'http-downgrade' => 'light_red', 'configuration-line' => 'red', 'content-type' => 'yellow', + 'HOST' => HEADER_DEFAULT_COLOUR, ); - our %h_colours = %h; + %h_colours = %h; # Header colours need their own hash so the keys can be accessed properly - our %header_colours = ( + %header_colours = ( # Prefilled with headers that should not appear with default header colours Cookie => 'light_red', 'Set-Cookie' => 'light_red', @@ -165,11 +197,11 @@ sub prepare_our_stuff () { ); # Crunch reasons need their own hash as well - our %reason_colours = ( + %reason_colours = ( 'Unsupported HTTP feature' => 'light_red', Blocked => 'light_red', Untrusted => 'light_red', - Redirected => 'green', + Redirected => 'green', 'CGI Call' => 'white', 'DNS failure' => 'red', 'Forwarding failed' => 'light_red', @@ -178,7 +210,7 @@ sub prepare_our_stuff () { 'No reason recorded' => 'light_red', ); - our @time_colours = ('white', 'light_gray'); + @time_colours = ('white', 'light_gray'); # Translate highlight strings into highlight code prepare_highlight_hash(\%header_colours); @@ -191,25 +223,25 @@ sub prepare_our_stuff () { init_stats(); } -sub paint_it ($) { +sub paint_it($) { ############################################################### # Takes a colour string and returns an ANSI escape sequence # (unless --no-syntax-highlighting is used). # XXX: The Rolling Stones reference has to go. ############################################################### - my $colour = shift @_; + my $colour = shift; return "" if cli_option_is_set('no-syntax-highlighting'); my %light = ( - black => 0, - red => 0, - green => 0, - brown => 0, - blue => 0, - purple => 0, - cyan => 0, + black => 0, + red => 0, + green => 0, + brown => 0, + blue => 0, + purple => 0, + cyan => 0, light_gray => 0, gray => 0, dark_gray => 1, @@ -223,13 +255,13 @@ sub paint_it ($) { ); my %text = ( - black => 30, - red => 31, - green => 32, - brown => 33, - blue => 34, - purple => 35, - cyan => 36, + black => 30, + red => 31, + green => 32, + brown => 33, + blue => 34, + purple => 35, + cyan => 36, gray => 37, light_gray => 37, dark_gray => 30, @@ -252,9 +284,9 @@ sub paint_it ($) { $colour_code .= ";"; $colour_code .= $light{$colour} ? "1" : "2"; $colour_code .= ";"; - $colour_code .= $bg_code; + $colour_code .= $bg_code; $colour_code .= "m"; - debug_message $colour . " is \'" . $colour_code . $colour . $default . "\'" if DEBUG_PAINT_IT; + debug_message $colour . " is \'" . $colour_code . $colour . $default . "\'" if DEBUG_PAINT_IT; } elsif ($colour =~ /reset/) { @@ -262,18 +294,18 @@ sub paint_it ($) { } else { - die "What's $colour supposed to mean?\n"; + die "What's $colour supposed to mean?\n"; } return $colour_code; } -sub get_semantic_html_markup ($) { +sub get_semantic_html_markup($) { ############################################################### # Takes a string and returns a span element ############################################################### - my $type = shift @_; + my $type = shift; my $code; if ($type =~ /Standard/) { @@ -286,7 +318,7 @@ sub get_semantic_html_markup ($) { return $code; } -sub cli_option_is_set ($) { +sub cli_option_is_set($) { our %cli_options; my $cli_option = shift; @@ -296,7 +328,7 @@ sub cli_option_is_set ($) { return $cli_options{$cli_option}; } -sub get_html_title () { +sub get_html_title() { our %cli_options; return $cli_options{'title'}; @@ -306,13 +338,13 @@ sub get_html_title () { sub init_css_colours() { our %css_colours = ( - black => "000", - red => "F00", - green => "0F0", - brown => "C90", - blue => "0F0", - purple => "F06", # XXX: wrong - cyan => "F09", # XXX: wrong + black => "000", + red => "F00", + green => "0F0", + brown => "C90", + blue => "0F0", + purple => "F06", # XXX: wrong + cyan => "F09", # XXX: wrong light_gray => "999", gray => "333", dark_gray => "222", @@ -326,35 +358,31 @@ sub init_css_colours() { ); } -sub get_css_colour ($) { +sub get_css_colour($) { our %css_colours; my $colour = shift; - die "What's $colour supposed to mean?\n" unless defined($css_colours{$colour}); + die "What's $colour supposed to mean?\n" unless defined($css_colours{$colour}); return '#' . $css_colours{$colour}; } -sub get_css_line ($) { - - our %h_colours; +sub get_css_line($) { my $class = shift; my $css_line; $css_line .= '.' . lc($class) . ' {'; # XXX: lc() shouldn't be necessary - die "What's $class supposed to mean?\n" unless defined($h_colours{$class}); + die "What's $class supposed to mean?\n" unless defined($h_colours{$class}); $css_line .= 'color:' . get_css_colour($h_colours{$class}) . ';'; $css_line .= 'background-color:' . get_css_colour(DEFAULT_BACKGROUND) . ';'; - $css_line .= '}' . "\n"; + $css_line .= '}' . "\n"; return $css_line; } -sub get_css_line_for_colour ($) { - - our %h_colours; +sub get_css_line_for_colour($) { my $colour = shift; my $css_line; @@ -362,27 +390,26 @@ sub get_css_line_for_colour ($) { $css_line .= '.' . lc($colour) . ' {'; # XXX: lc() shouldn't be necessary $css_line .= 'color:' . get_css_colour($colour) . ';'; $css_line .= 'background-color:' . get_css_colour(DEFAULT_BACKGROUND) . ';'; - $css_line .= '}' . "\n"; + $css_line .= '}' . "\n"; return $css_line; } # XXX: Wrong solution -sub get_missing_css_lines () { +sub get_missing_css_lines() { my $css_line; $css_line .= '.' . 'default' . ' {'; $css_line .= 'color:' . HEADER_DEFAULT_COLOUR . ';'; $css_line .= 'background-color:' . get_css_colour(DEFAULT_BACKGROUND) . ';'; - $css_line .= '}' . "\n"; + $css_line .= '}' . "\n"; return $css_line; } -sub get_css () { +sub get_css() { - our %h_colours; our %css_colours; #XXX: Wrong solution my $css = ''; @@ -390,8 +417,8 @@ sub get_css () { $css .= '.privoxy-log {'; $css .= 'color:' . get_css_colour(DEFAULT_TEXT_COLOUR) . ';'; $css .= 'background-color:' . get_css_colour(DEFAULT_BACKGROUND) . ';'; - $css .= '}' . "\n"; - + $css .= '}' . "\n"; + foreach my $key (keys %h_colours) { next if ($h_colours{$key} =~ m/reset/); #XXX: Wrong solution. @@ -410,7 +437,7 @@ sub get_css () { return $css; } -sub print_intro () { +sub print_intro() { my $intro = ''; @@ -428,7 +455,7 @@ sub print_intro () { } } -sub print_outro () { +sub print_outro() { my $outro = ''; @@ -440,23 +467,18 @@ sub print_outro () { } } -sub get_line_end () { - - my $line_end = "\n"; - - $line_end = '
' . $line_end if cli_option_is_set('html-output'); - - return $line_end; +sub get_line_end() { + return cli_option_is_set('html-output') ? "
\n" : "\n"; } -sub get_colour_html_markup ($) { +sub get_colour_html_markup($) { ############################################################### # Takes a colour string a span element. XXX: WHAT? # XXX: This function shouldn't be necessary, the # markup should always be semantically correct. ############################################################### - my $type = shift @_; + my $type = shift; my $code; if ($type =~ /Standard/) { @@ -468,36 +490,36 @@ sub get_colour_html_markup ($) { return $code; } -sub default_colours () { +sub default_colours() { # XXX: Properly our $bg_code; return reset_colours(); } -sub show_colours () { +sub show_colours() { # XXX: Implement } -sub reset_colours () { +sub reset_colours() { return ESCAPE . "0m"; } -sub set_background ($){ +sub set_background($) { my $colour = shift; our $bg_code; my %backgrounds = ( - black => "40", - red => "41", - green => "42", - brown => "43", - blue => "44", - magenta => "45", + black => "40", + red => "41", + green => "42", + brown => "43", + blue => "44", + magenta => "45", cyan => "46", - white => "47", - default => "49", + white => "47", + default => "49", ); - + if (defined($backgrounds{$colour})) { $bg_code = $backgrounds{$colour}; } else { @@ -505,55 +527,36 @@ sub set_background ($){ } } -sub get_background (){ +sub get_background() { return our $bg_code; } -sub prepare_highlight_hash ($) { +sub prepare_highlight_hash($) { my $ref = shift; - if (!cli_option_is_set('html-output')) { - - foreach my $key (keys %$ref) { - $$ref{$key} = paint_it($$ref{$key}); - } - - } else { - - foreach my $key (keys %$ref) { - $$ref{$key} = get_semantic_html_markup($key); - } - - } + foreach my $key (keys %$ref) { + $$ref{$key} = $html_output_mode ? + get_semantic_html_markup($key) : + paint_it($$ref{$key}); + } } -sub prepare_colour_array ($) { +sub prepare_colour_array($) { my $ref = shift; - if (!cli_option_is_set('html-output')) { - - foreach my $i (0 ... @$ref - 1) { - $$ref[$i] = paint_it($$ref[$i]); - } - - } else { - - foreach my $i (0 ... @$ref - 1) { - $$ref[$i] = get_colour_html_markup($$ref[$i]); - } - + foreach my $i (0 ... @$ref - 1) { + $$ref[$i] = $html_output_mode ? + get_colour_html_markup($$ref[$i]) : + paint_it($$ref[$i]); } } -sub found_unknown_content ($) { +sub found_unknown_content($) { my $unknown = shift; my $message; - our %req; - our $t; - - return if cli_option_is_set('accept-unknown-messages'); + return unless cli_option_is_set('strict-checks'); return if ($unknown =~ /\[too long, truncated\]$/); @@ -567,32 +570,30 @@ sub found_unknown_content ($) { die "Unworthy content parser" if PUNISH_MISSING_LOG_KNOWLEDGE_WITH_DEATH; } -sub log_parse_error ($) { +sub log_parse_error($) { my $message = shift; if (LOG_UNPARSED_LINES_TO_EXTRA_FILE) { - open(ERRORLOG, ">>" . ERROR_LOG_FILE) || die "Writing " . ERROR_LOG_FILE . " failed"; - print ERRORLOG $message; - close(ERRORLOG); + open(my $errorlog_fd, ">>", ERROR_LOG_FILE) || die "Writing " . ERROR_LOG_FILE . " failed"; + print $errorlog_fd $message; + close($errorlog_fd); } } -sub debug_message (@) { +sub debug_message(@) { my @message = @_; - our %h; print $h{'debug'} . "@message" . $h{'Standard'} . "\n"; } ################################################################################ -# highlighter functions that aren't loglevel-specific +# highlighter functions that aren't loglevel-specific ################################################################################ -sub h ($) { +sub h($) { # Get highlight marker - our %h; my $highlight = shift; # XXX: Stupid name; my $result = ''; my $message; @@ -608,29 +609,26 @@ sub h ($) { log_parser_error($message); die "Unworthy highlighter function" if PUNISH_MISSING_HIGHLIGHT_KNOWLEDGE_WITH_DEATH; } - + return $result; } -sub highlight_known_headers ($) { +sub highlight_known_headers($) { my $content = shift; - our %header_colours; - our %h; - my $headers = join ('|', keys %header_colours); debug_message("Searching $content for things to highlight.") if DEBUG_HEADER_HIGHLIGHTING; - if ($content =~ m/(?<=\s)($headers):/) { + if ($content =~ m/(?<=\s)($header_highlight_regex):/) { my $header = $1; $content =~ s@(?<=[\s|'])($header)(?=:)@$header_colours{$header}$1$h{'Standard'}@ig; - debug_message("Highlighted $content") if DEBUG_HEADER_HIGHLIGHTING; + debug_message("Highlighted '$header' in '$content'") if DEBUG_HEADER_HIGHLIGHTING; } return $content; } -sub highlight_matched_request_line ($$) { +sub highlight_matched_request_line($$) { my $result = shift; # XXX: Stupid name; my $regex = shift; @@ -640,18 +638,17 @@ sub highlight_matched_request_line ($$) { return $result; } -sub highlight_request_line ($) { +sub highlight_request_line($) { my $rl = shift; my ($method, $url, $http_version); - our %h; #GET http://images.sourceforge.net/sfx/icon_warning.gif HTTP/1.1 if ($rl =~ m/Invalid request/) { $rl = h('invalid-request') . $rl . h('Standard'); - } elsif ($rl =~ m/^([-\w]+) (.*) (HTTP\/\d\.\d)/) { + } elsif ($rl =~ m/^([-\w]+) (.*) (HTTP\/\d+\.\d+)/) { # XXX: might not match in case of HTTP method fuzzing. # XXX: save these: ($method, $path, $http_version) = ($1, $2, $3); @@ -682,7 +679,7 @@ sub highlight_request_line ($) { return $rl; } -sub highlight_response_line ($) { +sub highlight_response_line($) { my $rl = shift; my ($http_version, $status_code, $status_message); @@ -709,7 +706,7 @@ sub highlight_response_line ($) { return $rl; } -sub highlight_matched_url ($$) { +sub highlight_matched_url($$) { my $result = shift; # XXX: Stupid name; my $regex = shift; @@ -724,21 +721,19 @@ sub highlight_matched_url ($$) { return $result; } -sub highlight_matched_host ($$) { +sub highlight_matched_host($$) { - my $result = shift; # XXX: Stupid name; - my $regex = shift; + my ($result, $regex) = @_; # XXX: result ist stupid name; if ($result =~ m@(.*?)($regex)(.*)@) { - $result = $1 . h('host') . $2 . h('Standard') . $3; + $result = $1 . $h{host} . $2 . $h{Standard} . $3; } return $result; } -sub highlight_matched_pattern ($$$) { +sub highlight_matched_pattern($$$) { - our %h; my $result = shift; # XXX: Stupid name; my $key = shift; my $regex = shift; @@ -752,7 +747,7 @@ sub highlight_matched_pattern ($$$) { return $result; } -sub highlight_matched_path ($$) { +sub highlight_matched_path($$) { my $result = shift; # XXX: Stupid name; my $regex = shift; @@ -764,11 +759,11 @@ sub highlight_matched_path ($$) { return $result; } -sub highlight_url ($) { +sub highlight_url($) { my $url = shift; - if (cli_option_is_set('html-output')) { + if ($html_output_mode) { $url = '' . $url . ''; @@ -781,96 +776,96 @@ sub highlight_url ($) { return $url; } +sub update_header_highlight_regex($) { + + my $header = shift; + my $headers = join ('|', keys %header_colours); + + $header_highlight_regex = qr/$headers/; + print "Registering '$header'\n" if DEBUG_HEADER_HIGHLIGHTING; +} + ################################################################################ # loglevel-specific highlighter functions ################################################################################ -sub handle_loglevel_header ($) { +sub handle_loglevel_header($) { - my $content = shift; - my $c = $content; - our $t; - our %req; - our %h; - our %header_colours; - our @all_colours; - our $header_colour_index; - our $no_special_header_highlighting; - - # Register new headers - # scan: Accept: image/png,image/*;q=0.8,*/*;q=0.5 - if ($c =~ m/^scan: ((?>[^:]+)):/) { - my $header = $1; - if (!defined($header_colours{$header}) and $header =~ /^[\d\w-]*$/) { - debug_message "Registering previously unknown header $1" if DEBUG_HEADER_REGISTERING; + my $c = shift; - if (REGISTER_HEADERS_WITH_THE_SAME_COLOUR) { - $header_colours{$header} = $header_colours{'Default'}; - } else { - $header_colours{$header} = $all_colours[$header_colour_index % @all_colours]; - $header_colour_index++; - } - } - } + if ($c =~ /^scan:/) { + + if ($c =~ m/^scan: ([^: ]+):/) { - if ($c =~ m/^scan: ((\w*) (.*) (HTTP\/\d\.\d))/) { + # Register new headers + # scan: Accept: image/png,image/*;q=0.8,*/*;q=0.5 + my $header = $1; + if (!defined($header_colours{$header}) and $header =~ /^[\d\w-]*$/) { + debug_message "Registering previously unknown header $1" if DEBUG_HEADER_REGISTERING; - # Client request line - # Save for statistics (XXX: Not implemented yet) - $req{$t}{'method'} = $2; - $req{$t}{'destination'} = $3; - $req{$t}{'http-version'} = $4; + if (REGISTER_HEADERS_WITH_THE_SAME_COLOUR) { + $header_colours{$header} = $header_colours{'Default'}; + } else { + $header_colours{$header} = $all_colours[$header_colour_index % @all_colours]; + $header_colour_index++; + } + update_header_highlight_regex($header); + } + + } elsif ($c =~ m/^(scan: )(\w+ .+ HTTP\/\d\.\d)/) { - $content = highlight_request_line($1); + # scan: GET http://p.p/ HTTP/1.1 + $c = $1 . highlight_request_line($2); - } elsif ($c =~ m/^(scan: )((?:HTTP\/\d\.\d|ICY) (\d+) (.*))/) { + } elsif ($c =~ m/^(scan: )((?:HTTP\/\d\.\d|ICY) (\d+) (.*))/) { - # Server response line + # scan: HTTP/1.1 200 OK $req{$t}{'response_line'} = $2; $req{$t}{'status_code'} = $3; $req{$t}{'status_message'} = $4; - $content = $1 . highlight_response_line($req{$t}{'response_line'}); + $c = $1 . highlight_response_line($req{$t}{'response_line'}); + } } elsif ($c =~ m/^Crunching (?:server|client) header: .* \(contains: ([^\)]*)\)/) { # Crunching server header: Set-Cookie: trac_form_token=d5308c34e16d15e9e301a456; (contains: Cookie:) - $content =~ s@(?<=contains: )($1)@$h{'crunch-pattern'}$1$h{'Standard'}@; - $content =~ s@(Crunching)@$h{$1}$1$h{'Standard'}@; + $c =~ s@(?<=contains: )($1)@$h{'crunch-pattern'}$1$h{'Standard'}@; + $c =~ s@(Crunching)@$h{$1}$1$h{'Standard'}@; } elsif ($c =~ m/^New host is: ([^\s]*)\./) { - # New host is: trac.vidalia-project.net. Crunching Referer: http://www.vidalia-project.net/ - $c = highlight_matched_host($c, '(?<=New host is: )[^\s]+'); - $content = highlight_matched_url($c, '(?<=Crunching Referer: )[^\s]+'); + # New host is: trac.vidalia-project.net. Crunching Referer: http://www.vidalia-project.net/! + $c = highlight_matched_host($c, '(?<=New host is: )[^\s]+(?=\.)'); + $c = highlight_matched_url($c, '(?<=Crunching Referer: )[^\s!]+'); } elsif ($c =~ m/^Text mode enabled by force. (Take cover)!/) { # Text mode enabled by force. Take cover! - $content =~ s@($1)@$h{'warning'}$1$h{'Standard'}@; + $c =~ s@($1)@$h{'warning'}$1$h{'Standard'}@; } elsif ($c =~ m/^(New HTTP Request-Line: )(.*)/) { # New HTTP Request-Line: GET http://www.privoxy.org/ HTTP/1.1 - $content = $1 . highlight_request_line($2); + $c = $1 . highlight_request_line($2); } elsif ($c =~ m/^Adjust(ed)? Content-Length to \d+/) { # Adjusted Content-Length to 2132 # Adjust Content-Length to 33533 - $content =~ s@(?<=Content-Length to )(\d+)@$h{'Number'}$1$h{'Standard'}@; - $content = highlight_known_headers($content); + $c =~ s@(?<=Content-Length to )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $c = highlight_known_headers($c); } elsif ($c =~ m/^Destination extracted from "Host:" header. New request URL:/) { # Destination extracted from "Host:" header. New request URL: http://www.cccmz.de/~ridcully/blog/ - $content = highlight_matched_url($content, '(?<=New request URL: ).*'); + $c = highlight_matched_url($c, '(?<=New request URL: ).*'); } elsif ($c =~ m/^Couldn\'t parse:/) { # XXX: These should probable be logged with LOG_LEVEL_ERROR # Couldn't parse: If-Modified-Since: Wed, 21 Mar 2007 16:34:50 GMT (crunching!) # Couldn't parse: at, 24 Mar 2007 13:46:21 GMT in If-Modified-Since: Sat, 24 Mar 2007 13:46:21 GMT (crunching!) - $content =~ s@^(Couldn\'t parse)@$h{'error'}$1$h{'Standard'}@; + $c =~ s@^(Couldn\'t parse)@$h{'error'}$1$h{'Standard'}@; } elsif ($c =~ /^Tagger \'([^\']*)\' added tag \'([^\']*)\'/ or $c =~ m/^Adding tag \'([^\']*)\' created by header tagger \'([^\']*)\'/) { @@ -881,9 +876,9 @@ sub handle_loglevel_header ($) { # XXX: Save tag and tagger - $content =~ s@(?<=^Tagger \')([^\']*)@$h{'tagger'}$1$h{'Standard'}@; - $content =~ s@(?<=added tag \')([^\']*)@$h{'tag'}$1$h{'Standard'}@; - $content =~ s@(?<=Action bits )(updated)@$h{'action-bits-update'}$1$h{'Standard'}@; + $c =~ s@(?<=^Tagger \')([^\']*)@$h{'tagger'}$1$h{'Standard'}@; + $c =~ s@(?<=added tag \')([^\']*)@$h{'tag'}$1$h{'Standard'}@; + $c =~ s@(?<=Action bits )(updated)@$h{'action-bits-update'}$1$h{'Standard'}@; $no_special_header_highlighting = 1; } elsif ($c =~ /^Tagger \'([^\']*)\' didn['']t add tag \'([^\']*)\'/) { @@ -891,8 +886,8 @@ sub handle_loglevel_header ($) { # Tagger 'revalidation' didn't add tag 'REVALIDATION-REQUEST'. Tag already present # XXX: Save tag and tagger - $content =~ s@(?<=^Tagger \')([^\']*)@$h{'tag'}$1$h{'Standard'}@; - $content =~ s@(?<=didn['']t add tag \')([^\']*)@$h{'tagger'}$1$h{'Standard'}@; + $c =~ s@(?<=^Tagger \')([^\']*)@$h{'tag'}$1$h{'Standard'}@; + $c =~ s@(?<=didn['']t add tag \')([^\']*)@$h{'tagger'}$1$h{'Standard'}@; } elsif ($c =~ m/^(?:scan:|Randomiz|addh:|Adding:|Removing:|Referer:|Modified:|Accept-Language header|[Cc]ookie)/ or $c =~ m/^(Text mode is already enabled|Denied request with NULL byte|Replaced:|add-unique:)/ @@ -919,6 +914,11 @@ sub handle_loglevel_header ($) { or $c =~ m/^Appended client IP address to/ or $c =~ m/^Removing 'Connection: close' to imply keep-alive./ or $c =~ m/^keep-alive support is disabled/ + or $c =~ m/^Continue hack in da house/ + or $c =~ m/^Merged multiple header lines to:/ + or $c =~ m/^Added header: / + or $c =~ m/^Enlisting (?:sorted|left-over) header/ + or $c =~ m/^Multiple Content-Type headers detected. Removing and ignoring: Content-Type:/ ) { # XXX: Some of these may need highlighting @@ -964,6 +964,12 @@ sub handle_loglevel_header ($) { # Appended client IP address to X-Forwarded-For: 10.0.0.2, 10.0.0.1 # Removing 'Connection: close' to imply keep-alive. # keep-alive support is disabled. Crunching: Keep-Alive: 300. + # Continue hack in da house. + # Merged multiple header lines to: 'X-FORWARDED-PROTO: http X-HOST: 127.0.0.1' + # Added header: Content-Encoding: deflate + # Enlisting sorted header User-Agent: Mozilla/5.0 (X11; SunOS i86pc; rv:10.0.3) Gecko/20100101 Firefox/10.0.3 + # Enlisting left-over header Connection: close + # Multiple Content-Type headers detected. Removing and ignoring: Content-Type: text/html } elsif ($c =~ m/^scanning headers for:/) { @@ -973,21 +979,21 @@ sub handle_loglevel_header ($) { # crunched User-Agent! # Crunching: Content-Encoding: gzip - $content =~ s@(Crunching|crunched)@$h{$1}$1$h{'Standard'}@; + $c =~ s@(Crunching|crunched)@$h{$1}$1$h{'Standard'}@; } elsif ($c =~ m/^Offending request data with NULL bytes turned into \'°\' characters:/) { - + # Offending request data with NULL bytes turned into '°' characters: °°n°°(°°° - $content = h('warning') . $content . h('Standard'); - + $c = h('warning') . $c . h('Standard'); + } elsif ($c =~ m/^(Transforming \")(.*?)(\" to \")(.*?)(\")/) { # Transforming "Proxy-Authenticate: Basic realm="Correos Proxy Server"" to\ # "Proxy-Authenticate: Basic realm="Correos Proxy Server"" - $content =~ s@(?<=^Transforming \")(.*)(?=\" to)@$h{'Header'}$1$h{'Standard'}@; - $content =~ s@(?<=to \")(.*)(?=\")@$h{'Header'}$1$h{'Standard'}@; + $c =~ s@(?<=^Transforming \")(.*)(?=\" to)@$h{'Header'}$1$h{'Standard'}@; + $c =~ s@(?<=to \")(.*)(?=\")@$h{'Header'}$1$h{'Standard'}@; } elsif ($c =~ m/^Removing empty header/) { @@ -999,49 +1005,80 @@ sub handle_loglevel_header ($) { # Content-Type: application/octet-stream not replaced. It doesn't look like text.\ # Enable force-text-mode if you know what you're doing. # XXX: Could highlight more here. - $content =~ s@(?<=^Content-Type: )(.*)(?= not replaced)@$h{'content-type'}$1$h{'Standard'}@; + $c =~ s@(?<=^Content-Type: )(.*)(?= not replaced)@$h{'content-type'}$1$h{'Standard'}@; } elsif ($c =~ m/^(Server|Client) keep-alive timeout is/) { # Server keep-alive timeout is 5. Sticking with 10. # Client keep-alive timeout is 20. Sticking with 10. - $content =~ s@(?<=timeout is )(\d+)@$h{'Number'}$1$h{'Standard'}@; - $content =~ s@(?<=Sticking with )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $c =~ s@(?<=timeout is )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $c =~ s@(?<=Sticking with )(\d+)@$h{'Number'}$1$h{'Standard'}@; } elsif ($c =~ m/^Reducing keep-alive timeout/) { # Reducing keep-alive timeout from 60 to 10. - $content =~ s@(?<= from )(\d+)@$h{'Number'}$1$h{'Standard'}@; - $content =~ s@(?<= to )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $c =~ s@(?<= from )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $c =~ s@(?<= to )(\d+)@$h{'Number'}$1$h{'Standard'}@; + + } elsif ($c =~ m/^Killed all-caps Host header line: HOST:/) { + + # Killed all-caps Host header line: HOST: bestproxydb.com + $c = highlight_matched_host($c, '(?<=HOST: )[^\s]+'); + $c = highlight_matched_pattern($c, 'HOST', 'HOST'); } else { - found_unknown_content($content); + found_unknown_content($c); } - # Highlight headers + # Highlight headers unless ($c =~ m/^Transforming/) { - $content = highlight_known_headers($content) unless $no_special_header_highlighting; + $c = highlight_known_headers($c) unless $no_special_header_highlighting; } - return $content; + return $c; } -sub handle_loglevel_re_filter ($) { +sub handle_loglevel_re_filter($) { my $content = shift; my $c = $content; my $key; - our $t; - our %req; - our %h; - our %header_colours; - our @all_colours; - our $header_colour_index; - - if ($c =~ /\.{3}$/ + + if ($c =~ m/^(?:re_)?filtering ([^\s]+) \(size (\d+)\) with (?:filter )?\'?([^\s]+?)\'? produced (\d+) hits \(new size (\d+)\)/) { + + # XXX: only the second version gets highlighted properly. + # re_filtering www.lfk.de/favicon.ico (size 209) with filter untrackable-hulk produced 0 hits (new size 209). + # filtering aci.blogg.de/ (size 37988) with 'blogg.de' produced 3 hits (new size 38057) + $req{$t}{'content_source'} = $1; + $req{$t}{'content_size'} = $2; + $req{$t}{'content_filter'} = $3; + $req{$t}{'content_hits'} = $4; + $req{$t}{'new_content_size'} = $5; + $req{$t}{'content_size_change'} = $req{$t}{'new_content_size'} - $req{$t}{'content_size'}; + #return '' if ($req{$t}{'content_hits'} == 0 && !cli_option_is_set('show-ineffective-filters')); + if ($req{$t}{'content_hits'} == 0 and + not (cli_option_is_set('show-ineffective-filters') + or ($req{$t}{'content_filter'} =~ m/^privoxy-filter-test$/))) { + return ''; + } + + $c =~ s@(?<=\(size )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $c =~ s@(?<=\(new size )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $c =~ s@(?<=produced )(\d+)(?= hits)@$h{'Number'}$1$h{'Standard'}@; + + $c =~ s@([^\s]+?)(\'? produced)@$h{'filter'}$1$h{'Standard'}$2@; + $c = highlight_matched_host($c, '(?<=filtering )[^\s]+'); + + $c =~ s@\.$@ @; + $c .= "(" . $h{'Number'}; + $c .= "+" if ($req{$t}{'content_size_change'} >= 0); + $c .= $req{$t}{'content_size_change'} . $h{'Standard'} . ")"; + $content = $c; + + } elsif ($c =~ /\.{3}$/ and $c =~ m/^(?:re_)?filtering \'?(.*?)\'? \(size (\d*)\) with (?:filter )?\'?([^\s]*?)\'? ?\.{3}$/) { # Used by Privoxy 3.0.5 and 3.0.6: @@ -1057,7 +1094,7 @@ sub handle_loglevel_re_filter ($) { $req{$t}{'header_filter_name'} =~ m/^privoxy-filter-test$/) { return ''; } - $content =~ s@(?<=\(size )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $content =~ s@(?<=\(size )(\d+)@$h{'Number'}$1$h{'Standard'}@; $content =~ s@($req{$t}{'header_filter_name'})@$h{'filter'}$1$h{'Standard'}@; } elsif ($c =~ m/^ ?\.\.\. ?produced (\d*) hits \(new size (\d*)\)\./) { @@ -1075,7 +1112,7 @@ sub handle_loglevel_re_filter ($) { if ($req{$t}{'header_filter_hits'} == 0 and not (defined($req{$t}{'header_filter_name'}) and $req{$t}{'header_filter_name'} =~ m/^privoxy-filter-test$/)) { - return ''; + return ''; } # Reformat including information from the intro $c = "'" . h('filter') . $req{$t}{'header_filter_name'} . h('Standard') . "'"; @@ -1100,7 +1137,7 @@ sub handle_loglevel_re_filter ($) { } # Highlight from last line (XXX: What?) - # $c =~ s@(?<=produced )(\d+)@$h{'Number'}$1$h{'Standard'}@; + # $c =~ s@(?<=produced )(\d+)@$h{'Number'}$1$h{'Standard'}@; # $c =~ s@($req{$t}{'header_filter_name'})@$h{'filter'}$1$h{'Standard'}@; } else { @@ -1119,37 +1156,6 @@ sub handle_loglevel_re_filter ($) { $content =~ s@(?<=$1 )([^\s]*)@$h{'filter'}$1$h{'Standard'}@; - } elsif ($c =~ m/^(?:re_)?filtering ([^\s]+) \(size (\d+)\) with (?:filter )?\'?([^\s]+?)\'? produced (\d+) hits \(new size (\d+)\)/) { - - # XXX: only the second version gets highlighted properly. - # re_filtering www.lfk.de/favicon.ico (size 209) with filter untrackable-hulk produced 0 hits (new size 209). - # filtering aci.blogg.de/ (size 37988) with 'blogg.de' produced 3 hits (new size 38057) - $req{$t}{'content_source'} = $1; - $req{$t}{'content_size'} = $2; - $req{$t}{'content_filter'} = $3; - $req{$t}{'content_hits'} = $4; - $req{$t}{'new_content_size'} = $5; - $req{$t}{'content_size_change'} = $req{$t}{'new_content_size'} - $req{$t}{'content_size'}; - #return '' if ($req{$t}{'content_hits'} == 0 && !cli_option_is_set('show-ineffective-filters')); - if ($req{$t}{'content_hits'} == 0 and - not (cli_option_is_set('show-ineffective-filters') - or ($req{$t}{'content_filter'} =~ m/^privoxy-filter-test$/))) { - return ''; - } - - $c =~ s@(?<=\(size )(\d+)\)(?= with)@$h{'Number'}$1$h{'Standard'}@; - $c =~ s@(?<=\(new size )(\d+)@$h{'Number'}$1$h{'Standard'}@; - $c =~ s@(?<=produced )(\d+)(?= hits)@$h{'Number'}$1$h{'Standard'}@; - - $c =~ s@([^\s]+?)(\'? produced)@$h{'filter'}$1$h{'Standard'}$2@; - $c = highlight_matched_host($c, '(?<=filtering )[^\s]+'); - - $c =~ s@\.$@ @; - $c .= "(" . $h{'Number'}; - $c .= "+" if ($req{$t}{'content_size_change'} >= 0); - $c .= $req{$t}{'content_size_change'} . $h{'Standard'} . ")"; - $content = $c; - } elsif ($c =~ m/^De-chunking successful. Shrunk from (\d+) to (\d+)/) { $req{$t}{'chunked-size'} = $1; @@ -1197,10 +1203,23 @@ sub handle_loglevel_re_filter ($) { # Adding dynamic re_filter job s@^(?:\w*)\s+.*\s+HTTP/\d\.\d\s*@IP-ADDRESS: $origin@D\ # to filter client-ip-address succeeded. + } elsif ($c =~ m/^Compressed content from /) { + + # Compressed content from 29258 to 8630 bytes. Compression level: 3 + $content =~ s@(?<=from )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $content =~ s@(?<=to )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $content =~ s@(?<=level: )(\d+)@$h{'Number'}$1$h{'Standard'}@; + } elsif ($c =~ m/^Reading in filter/) { return '' unless SHOW_FILTER_READIN_IN; + } elsif ($c =~ m/^Decompression didn't result/) { + + # Decompression didn't result in any content. + + # Nothing to highlight. + } else { found_unknown_content($content); @@ -1210,12 +1229,44 @@ sub handle_loglevel_re_filter ($) { return $content; } -sub handle_loglevel_redirect ($) { +sub handle_loglevel_tagging($) { + + my $c = shift; + + if ($c =~ /^Tagger \'([^\']*)\' added tag \'([^\']*)\'/ or + $c =~ m/^Adding tag \'([^\']*)\' created by header tagger \'([^\']*)\'/) { + + # Adding tag 'GET request' created by header tagger 'method-man' (XXX: no longer used) + # Tagger 'revalidation' added tag 'REVALIDATION-REQUEST'. No action bit update necessary. + # Tagger 'revalidation' added tag 'REVALIDATION-REQUEST'. Action bits updated accordingly. + + # XXX: Save tag and tagger + + $c =~ s@(?<=^Tagger \')([^\']*)@$h{'tagger'}$1$h{'Standard'}@; + $c =~ s@(?<=added tag \')([^\']*)@$h{'tag'}$1$h{'Standard'}@; + $c =~ s@(?<=Action bits )(updated)@$h{'action-bits-update'}$1$h{'Standard'}@; + + } elsif ($c =~ /^Enlisting tag/) { + + # Enlisting tag 'forward-directly' for client 127.0.0.1. + + $c =~ s@(?<=tag \')([^\']*)@$h{'tag'}$1$h{'Standard'}@; + $c = highlight_matched_host($c, '[^\s]+(?=\.$)'); + + } elsif ($c =~ /^Client tag/) { + + # Client tag 'forward-directly' matches + + $c =~ s@(?<=tag \')([^\']*)@$h{'tag'}$1$h{'Standard'}@; + + } + + return $c; +} + +sub handle_loglevel_redirect($) { my $c = shift; - our $t; - our %req; - our %h; if ($c =~ m/^Decoding "([^""]*)"/) { @@ -1233,13 +1284,11 @@ sub handle_loglevel_redirect ($) { $c = highlight_matched_path($c, '(?<=Checking ")[^"]*'); $c =~ s@\"@@g; - } elsif ($c =~ m/^pcrs command "([^""]*)" changed "([^""]*)" to "([^""]*)" \((\d+) hits?\)/) { + } elsif ($c =~ m/^pcrs command "([^""]*)" changed /) { - # pcrs command "s@&from=rss@@" changed "http://it.slashdot.org/article.pl?sid=07/03/02/1657247&from=rss"\ + # pcrs command "s@&from=rss@@" changed \ + # "http://it.slashdot.org/article.pl?sid=07/03/02/1657247&from=rss"\ # to "http://it.slashdot.org/article.pl?sid=07/03/02/1657247" (1 hit). - - my ($pcrs_command, $url_before, $url_after, $hits) = ($1, $2, $3, $4); # XXX: save these? - $c =~ s@(?<=pcrs command )"([^""]*)"@$h{'filter'}$1$h{'Standard'}@; $c = highlight_matched_url($c, '(?<=changed ")[^""]*'); $c =~ s@(?<=changed )"([^""]*)"@$1@; # Remove quotes @@ -1247,6 +1296,13 @@ sub handle_loglevel_redirect ($) { $c =~ s@(?<=to )"([^""]*)"@$1@; # Remove quotes $c =~ s@(\d+)(?= hits?)@$h{'hits'}$1$h{'Standard'}@; + } elsif ($c =~ m/^pcrs command "([^""]*)" didn\'t change/) { + + # pcrs command "s@^http://([^.]+?)/?$@http://www.bing.com/search?q=$1@" didn't \ + # change "http://www.example.org/". + $c =~ s@(?<=pcrs command )"([^""]*)"@$h{'filter'}$1$h{'Standard'}@; + $c = highlight_matched_url($c, '(?<=change ")[^""]*'); + } elsif ($c =~ m/(^New URL is: )(.*)/) { # New URL is: http://it.slashdot.org/article.pl?sid=07/03/04/1511210 @@ -1260,6 +1316,22 @@ sub handle_loglevel_redirect ($) { # XXX: assume the same? $c = highlight_matched_url($c, '(?<=assuming that \")[^"]*'); + } elsif ($c =~ m/^Percent-encoding redirect/) { + + # Percent-encoding redirect URL: http://www.example.org/\x02 + $c = highlight_matched_url($c, '(?<=redirect URL: ).*'); + + } elsif ($c =~ m/^Rewrite detected:/) { + + # Rewrite detected: GET http://10.0.0.2:88/blah.txt HTTP/1.1 + # Rewrite detected: GET https://www.electrobsd.org/CommonJS/ajax/libs/jquery/3.4.1/jquery.min.js HTTP/1.1 + $c = highlight_matched_request_line($c, '(?<=^Rewrite detected: ).*'); + + } elsif ($c =~ m/^Rewritten request line results in downgrade to http/) { + + # Rewritten request line results in downgrade to http + $c =~ s@(downgrade)@$h{'http-downgrade'}$1$h{'Standard'}@; + } else { found_unknown_content($c); @@ -1269,12 +1341,9 @@ sub handle_loglevel_redirect ($) { return $c; } -sub handle_loglevel_gif_deanimate ($) { +sub handle_loglevel_gif_deanimate($) { my $content = shift; - our $t; - our %req; - our %h; if ($content =~ m/Success! GIF shrunk from (\d+) bytes to (\d+)\./) { @@ -1294,7 +1363,7 @@ sub handle_loglevel_gif_deanimate ($) { } elsif ($content =~ m/^failed! \(gif parsing\)/) { # failed! (gif parsing) - # XXX: Replace this error message with something less stupid + # XXX: Replace this error message with something less stupid $content =~ s@(failed!)@$h{'error'}$1$h{'Standard'}@; } elsif ($content =~ m/^Need to de-chunk first/) { @@ -1305,7 +1374,7 @@ sub handle_loglevel_gif_deanimate ($) { } elsif ($content =~ m/^(?:No GIF header found|failed while parsing)/) { # No GIF header found (XXX: Did I ever commit this?) - # failed while parsing 195 134747048 (XXX: never commited) + # failed while parsing 195 134747048 (XXX: never committed) # Ignore these for now @@ -1318,13 +1387,9 @@ sub handle_loglevel_gif_deanimate ($) { return $content; } -sub handle_loglevel_request ($) { +sub handle_loglevel_request($) { my $content = shift; - our $t; - our %req; - our %h; - our %reason_colours; if ($content =~ m/crunch! /) { @@ -1353,15 +1418,13 @@ sub handle_loglevel_request ($) { found_unknown_content($content); } - + return $content; } -sub handle_loglevel_crunch ($) { +sub handle_loglevel_crunch($) { my $content = shift; - our %h; - our %reason_colours; # Highlight crunch reason foreach my $reason (keys %reason_colours) { @@ -1374,6 +1437,11 @@ sub handle_loglevel_crunch ($) { # [...]&filter... [too long, truncated] $content = highlight_matched_pattern($content, 'request_', '^.*(?=\.\.\. \[too long, truncated\]$)'); + } elsif ($content =~ m/Certificate error:/) { + + # Certificate error: ASN date error, current date after: https://expired.badssl.com/ + $content = highlight_matched_pattern($content, 'request_', 'https://.*'); + } else { # Blocked: http://ads.example.org/ @@ -1383,12 +1451,9 @@ sub handle_loglevel_crunch ($) { return $content; } -sub handle_loglevel_connect ($) { +sub handle_loglevel_connect($) { my $c = shift; - our $t; - our %req; - our %h; if ($c =~ m/^via [^\s]+ to: [^\s]+/) { @@ -1419,14 +1484,31 @@ sub handle_loglevel_connect ($) { $c = highlight_matched_host($c, '(?<=to )[^\s]+'); - } elsif ($c =~ m/^accepted connection from .*/ or + } elsif ($c =~ m/^[Aa]ccepted connection from .*/ or $c =~ m/^OK/) { - # accepted connection from 10.0.0.1 + # Privoxy 3.0.20: + # Accepted connection from 10.0.0.1 on socket 5 + # Privoxy between 3.0.20 and 3.0.6: + # accepted connection from 10.0.0.1( on socket 5)? # Privoxy 3.0.6 and earlier just say: # OK - return '' if SUPPRESS_ACCEPTED_CONNECTIONS; - $c = highlight_matched_host($c, '(?<=connection from ).*'); + $c = highlight_matched_host($c, '(?<=connection from )[^ ]*'); + $c = highlight_matched_pattern($c, 'Number', '(?<=socket )\d+'); + + } elsif ($c =~ m/^Closing client socket/) { + + # Closing client socket 5. Keep-alive: 0, Socket alive: 1. Data available: 0. + # Privoxy 3.0.20 and later + # Closing client socket 8. Keep-alive: 1. Socket alive: 0. Data available: 0. \ + # Configuration file change detected: 0. Requests received: 11. + + $c = highlight_matched_pattern($c, 'Number', '(?<=socket )\d+'); + $c = highlight_matched_pattern($c, 'Number', '(?<=Keep-alive: )\d+'); + $c = highlight_matched_pattern($c, 'Number', '(?<=Socket alive: )\d+'); + $c = highlight_matched_pattern($c, 'Number', '(?<=available: )\d+'); + $c = highlight_matched_pattern($c, 'Number', '(?<=detected: )\d+'); + $c = highlight_matched_pattern($c, 'Number', '(?<=received: )\d+'); } elsif ($c =~ m/^write header to: .* failed:/) { @@ -1470,7 +1552,7 @@ sub handle_loglevel_connect ($) { $c = highlight_matched_host($c, '(?<=from )[^\s]+'); # XXX: not an URL } elsif ($c =~ m/^socks5_connect:/) { - + $c =~ s@(?<=socks5_connect: )(.*)@$h{'error'}$1$h{'Standard'}@; } elsif ($c =~ m/^Created new connection to/) { @@ -1531,10 +1613,17 @@ sub handle_loglevel_connect ($) { $c =~ s@(?<=Timeout is: )(\d+)@$h{'Number'}$1$h{'Standard'}@; $c =~ s@(?<=Assumed latency: )(\d+)@$h{'Number'}$1$h{'Standard'}@; - } elsif ($c =~ m/^Stopped waiting for the request line./) { + } elsif ($c =~ m/^Stopped waiting for the request line/ or + $c =~ m/^No request line on socket \d received in time/ or + $c =~ m/^The client side of the connection on socket \d/) { # Stopped waiting for the request line. Timeout: 121. + # Privoxy 3.0.19 and later: + # No request line on socket 5 received in time. Timeout: 1. + # The client side of the connection on socket 5 got closed \ + # without sending a complete request line. $c =~ s@(?<=Timeout: )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $c =~ s@(?<=socket )(\d+)@$h{'Number'}$1$h{'Standard'}@; } elsif ($c =~ m/^Waiting for \d/) { @@ -1559,15 +1648,18 @@ sub handle_loglevel_connect ($) { $c =~ s@(?<=received: )(\d+)@$h{'Number'}$1$h{'Standard'}@; $c =~ s@(?<=read: )(\d+)@$h{'Number'}$1$h{'Standard'}@; - } elsif ($c =~ m/^Continuing buffering headers/) { + } elsif ($c =~ m/^Continuing buffering (?:server )?headers/) { # Continuing buffering headers. byte_count: 19. header_offset: 517. len: 536. $c =~ s@(?<=byte_count: )(\d+)@$h{'Number'}$1$h{'Standard'}@; $c =~ s@(?<=header_offset: )(\d+)@$h{'Number'}$1$h{'Standard'}@; $c =~ s@(?<=len: )(\d+)@$h{'Number'}$1$h{'Standard'}@; - # 3.0.15 and later: - # Continuing buffering headers. Bytes most recently read: %d. + # 3.0.15 up to 3.0.19: + # Continuing buffering headers. Bytes most recently read: 498. $c =~ s@(?<=read: )(\d+)@$h{'Number'}$1$h{'Standard'}@; + # 3.0.20 and later: + # Continuing buffering server headers from socket 5. Bytes most recently read: 498. + $c =~ s@(?<=socket )(\d+)@$h{'Number'}$1$h{'Standard'}@; } elsif ($c =~ m/^Received \d+ bytes while/) { @@ -1575,21 +1667,36 @@ sub handle_loglevel_connect ($) { $c =~ s@(?<=Received )(\d+)@$h{'Number'}$1$h{'Standard'}@; $c =~ s@(?<=expecting )(\d+)@$h{'Number'}$1$h{'Standard'}@; - } elsif ($c =~ m/^Connection from/) { + } elsif ($c =~ m/^(Rejecting c|C)onnection from/) { # Connection from 81.163.28.218 dropped due to ACL - $c =~ s@(?<=^Connection from )((?:\d+\.?){4})@$h{'Number'}$1$h{'Standard'}@; + # Rejecting connection from 178.63.152.227. Maximum number of connections reached. + # Connection from 192.168.2.1 on 127.0.1.1:8118 (socket 3) dropped due to ACL + $c = highlight_matched_host($c, '(?<=onnection from )[\d.:]+'); + $c = highlight_matched_host($c, '(?<=on )[\d.:]+'); + $c =~ s@(?<=socket )(\d+)@$h{'Number'}$1$h{'Standard'}@; - } elsif ($c =~ m/^(?:Reusing|Closing) server socket \d./ or + } elsif ($c =~ m/^(?:Reusing|Closing) server socket / or $c =~ m/^No additional client request/) { # Reusing server socket 4. Opened for 10.0.0.1. # Closing server socket 2. Opened for 10.0.0.1. # No additional client request received in time. \ # Closing server socket 4, initially opened for 10.0.0.1. - - $c =~ s@(?<=server socket )(\d+)@$h{'Number'}$1$h{'Standard'}@; - $c = highlight_matched_host($c, '(?<=for )[^\s]+(?=\.$)'); + # No additional client request received in time on socket 29. + # Privoxy 3.0.20 and later + # Reusing server socket 7 connected to www.privoxy.org. Total requests: 2. + # Closing server socket 6 connected to d.asset.soup.io. Keep-alive: 0.\ + # Tainted: 1. Socket alive: 1. Timeout: 60. Configuration file change detected: 0. + # Reusing server socket 35 connected to nl.wikipedia.org. Requests already sent: 5. + + $c =~ s@(?<= socket )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $c = highlight_matched_host($c, '(?<=for )[^\s]+(?=\.)'); + $c = highlight_matched_host($c, '(?<=connected to )[^\s]+(?=\.)'); + for my $number_pattern ('requests', 'Keep-alive', 'Tainted', ' alive', 'Timeout', 'detected') { + $c = highlight_matched_pattern($c, 'Number', '(?<='. $number_pattern . ': )\d+'); + } + $c =~ s@(?<=already sent: )(\d+)@$h{'Number'}$1$h{'Standard'}@; } elsif ($c =~ m/^Connected to /) { @@ -1607,11 +1714,22 @@ sub handle_loglevel_connect ($) { $c =~ s@(?<=\]:)(\d+)@$h{'Number'}$1$h{'Standard'}@; } elsif ($c =~ m/^Waiting for the next client request/ or - $c =~ m/^The connection on server socket/ ) { + $c =~ m/^The connection on server socket/ or + $c =~ m/^Client request (?:\d+ )?(?:arrived in time|has been pipelined) /) { - # Waiting for the next client request. Keeping the server socket 5 to 10.0.0.1 open. + # Waiting for the next client request on socket 3. Keeping the server \ + # socket 12 to a.fsdn.com open. # The connection on server socket 6 to upload.wikimedia.org isn't reusable. Closing. - + # Privoxy 3.0.20 and later: + # Client request 4 arrived in time on socket 7. + # Used by Privoxy 3.0.18 and 3.0.19: + # Client request arrived in time on socket 21. + # Used by earlier version: + # Client request arrived in time or the client closed the connection on socket 12. + # Client request 8 has been pipelined on socket 7 and the socket is still alive. + + $c =~ s@(?<=request )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $c =~ s@(?<=on socket )(\d+)@$h{'Number'}$1$h{'Standard'}@; $c =~ s@(?<=server socket )(\d+)@$h{'Number'}$1$h{'Standard'}@; $c = highlight_matched_host($c, '(?<=to )[^\s]+'); @@ -1639,27 +1757,120 @@ sub handle_loglevel_connect ($) { $c =~ s@(?<=set to )(\d+)@$h{'Number'}$1$h{'Standard'}@; $c =~ s@(?<=reading )(\d+)@$h{'Number'}$1$h{'Standard'}@; + } elsif ($c =~ m/^Reducing expected bytes to /) { + + # Reducing expected bytes to 0. Marking the server socket tainted after throwing 4 bytes away. + $c =~ s@(?<=bytes to )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $c =~ s@(?<=after throwing )(\d+)@$h{'Number'}$1$h{'Standard'}@; + } elsif ($c =~ m/^Waiting for up to /) { # Waiting for up to 4999 bytes from the client. $c =~ s@(?<=up to )(\d+)@$h{'Number'}$1$h{'Standard'}@; - } elsif ($c =~ m/^Looks like we rea/ or + } elsif ($c =~ m/^Optimistically sending /) { + + # Optimistically sending 318 bytes of client headers intended for www.privoxy.org + $c =~ s@(?<=sending )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $c = highlight_matched_host($c, '(?<=for )[^\s]+'); + + } elsif ($c =~ m/^Stopping to watch the client socket/) { + + # Stopping to watch the client socket. There's already another request waiting. + # Privoxy 3.0.20 and later: + # Stopping to watch the client socket 5. There's already another request waiting. + $c =~ s@(?<=client socket )(\d+)@$h{'Number'}$1$h{'Standard'}@; + + } elsif ($c =~ m/^Drained \d+ bytes before closing/) { + + # Drained 180 bytes before closing socket 6 + $c =~ s@(?<=Drained )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $c =~ s@(?<=socket )(\d+)@$h{'Number'}$1$h{'Standard'}@; + + } elsif ($c =~ m/^Tainting client socket/ or + $c =~ m/^Failed to shutdown socket/) { + + # Tainting client socket 7 due to unread data. + # Failed to shutdown socket 11: Connection reset by peer + + $c =~ s@(?<=socket )(\d+)@$h{'Number'}$1$h{'Standard'}@; + + } elsif ($c =~ m/^Shifting \d+ pipelined bytes/) { + + # Shifting 360 pipelined bytes by 360 bytes + $c =~ s@(?<=Shifting )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $c =~ s@(?<=by )(\d+)@$h{'Number'}$1$h{'Standard'}@; + + } elsif ($c =~ m/^Flushed (\d+) bytes of request body while expecting (\d+)/) { + + # Flushed 30 bytes of request body while expecting 30 + $c =~ s@(?<=Flushed )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $c =~ s@(?<=expecting )(\d+)@$h{'Number'}$1$h{'Standard'}@; + + } elsif ($c =~ m/^Performing the TLS\/SSL handshake with client. Hash of host:/) { + + # Performing the TLS/SSL handshake with client. Hash of host: bab5296b25e256c7b06b92b17b56bcae + $c = highlight_matched_host($c, '(?<=Hash of host: ).+'); + + } elsif ($c =~ m/^Forwarding \d+ bytes of encrypted POST data/) { + + # Forwarding 1954 bytes of encrypted POST data + $c =~ s@(?<=Forwarding )(\d+)@$h{'Number'}$1$h{'Standard'}@; + + } elsif ($c =~ m/^Forwarded the last \d+ bytes/) { + + # Forwarded the last 1954 bytes + $c =~ s@(?<=the last )(\d+)@$h{'Number'}$1$h{'Standard'}@; + + } elsif ($c =~ m/^Waiting for the next client connection. Currently active threads:/) { + + # Waiting for the next client connection. Currently active threads: 30 + $c =~ s@(?<=threads: )(\d+)@$h{'Number'}$1$h{'Standard'}@; + + } elsif ($c =~ m/^Data arrived in time on client socket/) { + + # Data arrived in time on client socket 6. Requests so far: 3 + $c =~ s@(?<=client socket )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $c =~ s@(?<=Requests so far: )(\d+)@$h{'Number'}$1$h{'Standard'}@; + + } elsif ($c =~ m/^Dropping the client connection on socket/) { + + # Dropping the client connection on socket 71. The server connection has not been established yet. + $c =~ s@(?<=on socket )(\d+)@$h{'Number'}$1$h{'Standard'}@; + + } elsif ($c =~ m/^The client socket \d+ has become unusable while the server/) { + + # The client socket 16 has become unusable while the server socket 24 is still open. + $c =~ s@(?<=client socket )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $c =~ s@(?<=server socket )(\d+)@$h{'Number'}$1$h{'Standard'}@; + + } elsif ($c =~ m/^The last \d+ bytes of the request body have been read/) { + + # The last 12078 bytes of the request body have been read + $c =~ s@(?<=The last )(\d+)@$h{'Number'}$1$h{'Standard'}@; + + } elsif ($c =~ m/^Flushed \d+ bytes of request body/) { + + # Flushed 3153 bytes of request body + $c =~ s@(?<=Flushed )(\d+)@$h{'Number'}$1$h{'Standard'}@; + + } elsif ($c =~ m/^Looks like we / or $c =~ m/^Unsetting keep-alive flag/ or $c =~ m/^No connections to wait/ or - $c =~ m/^Client request arrived in time or the client closed the connection/ or $c =~ m/^Complete client request received/ or $c =~ m/^Possible pipeline attempt detected./ or $c =~ m/^POST request detected. The connection will not be kept alive./ or $c =~ m/^The server still wants to talk, but the client hung up on us./ or $c =~ m/^The server didn't specify how long the connection will stay open/ or $c =~ m/^There might be a request body. The connection will not be kept alive/ or - $c =~ m/^Stopping to watch the client socket. There's already another request waiting./ or + $c =~ m/^There better be a request body./ or $c =~ m/^Done reading from the client\.$/) { # Looks like we reached the end of the last chunk. We better stop reading. # Looks like we read the end of the last chunk together with the server \ # headers. We better stop reading. + # Looks like we got the last chunk together with the server headers. \ + # We better stop reading. # Unsetting keep-alive flag. # No connections to wait for left. # Client request arrived in time or the client closed the connection. @@ -1670,26 +1881,24 @@ sub handle_loglevel_connect ($) { # The server still wants to talk, but the client hung up on us. # The server didn't specify how long the connection will stay open. Assume it's only a second. # There might be a request body. The connection will not be kept alive. - # Stopping to watch the client socket. There's already another request waiting. - # Done reading from the client\. + # Privoxy 3.0.20 and later + # There better be a request body. + # Done reading from the client. } else { found_unknown_content($c); } - + return $c; } -sub handle_loglevel_info ($) { +sub handle_loglevel_info($) { my $c = shift; - our $t; - our %req; - our %h; - + if ($c =~ m/^Rewrite detected:/) { # Rewrite detected: GET http://10.0.0.2:88/blah.txt HTTP/1.1 @@ -1697,12 +1906,15 @@ sub handle_loglevel_info ($) { } elsif ($c =~ m/^Decompress(ing deflated|ion didn)/ or $c =~ m/^Compressed content detected/ or + $c =~ m/^SDCH-compressed content detected/ or $c =~ m/^Tagger/ ) { # Decompressing deflated iob: 117 # Decompression didn't result in any content. # Compressed content detected, content filtering disabled. Consider recompiling Privoxy\ # with zlib support or enable the prevent-compression action. + # SDCH-compressed content detected, content filtering disabled.\ + # Consider suppressing SDCH offers made by the client. # Tagger 'complete-url' created empty tag. Ignored. # Ignored for now @@ -1713,13 +1925,21 @@ sub handle_loglevel_info ($) { # Reloading configuration file '/usr/local/etc/privoxy/config' $c =~ s@(?<=loading configuration file \')([^\']*)@$h{'file'}$1$h{'Standard'}@; + } elsif ($c =~ m/^Loading (actions|filter|trust) file: /) { + + # Loading actions file: /usr/local/etc/privoxy/default.action + # Loading filter file: /usr/local/etc/privoxy/default.filter + # Loading trust file: /usr/local/etc/privoxy/trust + + $c =~ s@(?<= file: )(.*)$@$h{'file'}$1$h{'Standard'}@; + } elsif ($c =~ m/^exiting by signal/) { - + # exiting by signal 15 .. bye $c =~ s@(?<=exiting by signal )(\d+)@$h{'signal'}$1$h{'Standard'}@; } elsif ($c =~ m/^Privoxy version/) { - + # Privoxy version 3.0.7 $c =~ s@(?<=^Privoxy version )(\d+\.\d+\.\d+)$@$h{'version'}$1$h{'Standard'}@; @@ -1743,12 +1963,15 @@ sub handle_loglevel_info ($) { # Request from 10.0.0.1 denied. limit-connect{,} doesn't allow CONNECT requests to port 443. # Request from 10.0.0.1 marked for blocking. limit-connect{,} doesn't allow CONNECT requests to port 443. + # 3.0.18 and later: + # Request from 10.0.0.1 marked for blocking. limit-connect{0} doesn't allow CONNECT requests to www.example.org:443 # Malformed server response detected. Downgrading to HTTP/1.0 impossible. $c =~ s@(?<=Request from )([^\s]*)@$h{'ip-address'}$1$h{'Standard'}@; $c =~ s@(denied|blocking)@$h{'warning'}$1$h{'Standard'}@; $c =~ s@(CONNECT)@$h{'method'}$1$h{'Standard'}@; $c =~ s@(?<=to port )(\d+)@$h{'port'}$1$h{'Standard'}@; + $c =~ s@(?<=to )([^\s]+)@$h{'request_'}$1$h{'Standard'}@; } elsif ($c =~ m/^Status code/) { @@ -1766,11 +1989,17 @@ sub handle_loglevel_info ($) { $c =~ s@(?<=Needed: )(\d+)@$h{'Number'}$1$h{'Standard'}@; $c =~ s@(?<=Limit: )(\d+)@$h{'Number'}$1$h{'Standard'}@; + } elsif ($c =~ m/^File modification detected: /) { + + # File modification detected: /usr/local/etc/privoxy/user-agent.action + $c =~ s@(?<= detected: )(.*)$@$h{'file'}$1$h{'Standard'}@; + } elsif ($c =~ m/^No logfile configured/ or $c =~ m/^Malformerd HTTP headers detected and MS IIS5 hack enabled/ or $c =~ m/^Invalid \"chunked\" transfer/ or $c =~ m/^Support for/ or - $c =~ m/^Flushing header and buffers/ + $c =~ m/^Flushing header and buffers/ or + $c =~ m/^Can not resolve/ ) { # No logfile configured. Please enable it before reporting any problems. @@ -1781,6 +2010,7 @@ sub handle_loglevel_info ($) { # Support for 'Connection: keep-alive' is experimental, incomplete and\ # known not to work properly in some situations. # Flushing header and buffers. Stepping back from filtering. + # Can not resolve doesnotexist: hostname nor servname provided, or not known } else { @@ -1791,19 +2021,16 @@ sub handle_loglevel_info ($) { return $c; } -sub handle_loglevel_cgi ($) { +sub handle_loglevel_cgi($) { my $c = shift; - our $t; - our %req; - our %h; if ($c =~ m/^Granting access to/) { - + #Granting access to http://config.privoxy.org/send-stylesheet, referrer http://p.p/ is trustworthy. } elsif ($c =~ m/^Substituting: s(.)/) { - + # Substituting: s/@else-not-FEATURE_ZLIB@.*@endif-FEATURE_ZLIB@//sigTU # XXX: prone to span several lines @@ -1816,21 +2043,18 @@ sub handle_loglevel_cgi ($) { return $c; } -sub handle_loglevel_force ($) { +sub handle_loglevel_force($) { my $c = shift; - our $t; - our %req; - our %h; if ($c =~ m/^Ignored force prefix in request:/) { - + # Ignored force prefix in request: "GET http://10.0.0.1/PRIVOXY-FORCE/block HTTP/1.1" $c =~ s@^(Ignored)@$h{'ignored'}$1$h{'Standard'}@; $c = highlight_matched_request_line($c, '(?<=request: ")[^"]*'); } elsif ($c =~ m/^Enforcing request:/) { - + # Enforcing request: "GET http://10.0.0.1/block HTTP/1.1". $c = highlight_matched_request_line($c, '(?<=request: ")[^"]*'); @@ -1843,27 +2067,88 @@ sub handle_loglevel_force ($) { return $c; } -sub handle_loglevel_error ($) { +sub handle_loglevel_error($) { my $c = shift; - our %h; - if ($c =~ m/^Empty server or forwarder response received on socket \d+./) { + if ($c =~ m/^(?:Empty|No) server or forwarder response received on socket \d+\./) { # Empty server or forwarder response received on socket 4. + # Empty server or forwarder response received on socket 3. \ + # Closing client socket 15 without sending data. + # Used by Privoxy 3.0.18 and later: + # No server or forwarder response received on socket 8. \ + # Closing client socket 10 without sending data. + + $c =~ s@(?<=on socket )(\d+)@$h{'Number'}$1$h{'Standard'}@; + $c =~ s@(?<=client socket )(\d+)@$h{'Number'}$1$h{'Standard'}@; + + } elsif ($c =~ m/^Didn't receive data in time:/) { + + # Didn't receive data in time: a.fsdn.com:443 + $c =~ s@(?<=in time: )(.*)@$h{'destination'}$1$h{'Standard'}@; + + } elsif ($c =~ m/^Sending data on socket \d+ over TLS/) { + + # Sending data on socket 33 over TLS/SSL failed: no TLS/SSL errors detected $c =~ s@(?<=on socket )(\d+)@$h{'Number'}$1$h{'Standard'}@; + } + # XXX: There are probably more messages that deserve highlighting. return $c; } -sub handle_loglevel_ignore ($) { +sub handle_loglevel_ignore($) { return shift; } -sub gather_loglevel_request_stats ($$) { +sub gather_loglevel_clf_stats($) { + + my $content = shift; + my ($method, $resource, $http_version, $status_code, $size); + our %stats; + our %cli_options; + + # +0200] "GET https://www.youtube.com/watch?v=JmcA9LIIXWw HTTP/1.1" 200 68004 + # +0200] "VERSION-CONTROL http://p.p/ HTTP/1.1" 200 2787 + $content =~ m/^[+-]\d{4}\] "([^ ]+) (.+) (HTTP\/\d\.\d)" (\d+) (\d+)/; + $method = $1; + $resource = $2; + $http_version = $3; + $status_code = $4; + $size = $5; + + $stats{requests_clf}++; + + unless (defined $method) { + # +0200] "Invalid request" 400 0 + return if ($content =~ m/^[+-]\d{4}\] "Invalid request"/); + # +0100] "GET https://securepubads.g.doubleclick.net/gampad/ads?gd[...]... [too long, truncated] + if ($content =~ m/\[too long, truncated\]$/) { + print("Skipped LOG_LEVEL_CLF message that got truncated by Privoxy. Statistics will be inprecise.\n"); + } else { + print("Failed to parse: $content\n"); + } + return; + } + $stats{'method'}{$method}++; + if ($cli_options{'url-statistics-threshold'} != 0) { + $stats{'resource'}{$resource}++; + } + $stats{'http-version'}{$http_version}++; + + if ($cli_options{'host-statistics-threshold'} != 0) { + $resource =~ m@(?:https?://)?([^/]+)/?@; + $stats{'hosts'}{$1}++; + } + $stats{'content-size-total'} += $size; + $stats{'status-code'}{$status_code}++; +} + +sub gather_loglevel_request_stats($$) { my $c = shift; my $thread = shift; our %stats; @@ -1871,17 +2156,33 @@ sub gather_loglevel_request_stats ($$) { $stats{requests}++; } -sub gather_loglevel_crunch_stats ($$) { +sub gather_loglevel_crunch_stats($$) { my $c = shift; my $thread = shift; our %stats; - $stats{requests}++; $stats{crunches}++; + + if ($c =~ m/^Redirected:/) { + # Redirected: http://www.example.org/http://p.p/ + $stats{'fast-redirections'}++; + + } elsif ($c =~ m/^Blocked:/) { + # Blocked: blogger.googleusercontent.com:443 + $stats{'blocked'}++; + + } elsif ($c =~ m/^Connection timeout:/) { + # Connection timeout: http://c.tile.openstreetmap.org/18/136116/87842.png + $stats{'connection-timeout'}++; + + } elsif ($c =~ m/^Connection failure:/) { + # Connection failure: http://127.0.0.1:8080/ + $stats{'connection-failure'}++; + } } -sub gather_loglevel_error_stats ($$) { +sub gather_loglevel_error_stats($$) { my $c = shift; my $thread = shift; @@ -1900,10 +2201,9 @@ sub gather_loglevel_error_stats ($$) { } } -sub gather_loglevel_connect_stats ($$) { +sub gather_loglevel_connect_stats($$) { - my $c = shift; - my $thread = shift; + my ($c, $thread) = @_; our %thread_data; our %stats; @@ -1932,14 +2232,22 @@ sub gather_loglevel_connect_stats ($$) { $thread_data{$thread}{'new_connection'} = 0; $stats{'reused-connections'}++; + + } elsif ($c =~ m/^Closing client socket \d+. .* Requests received: (\d+)\.$/) { + + # Closing client socket 12. Keep-alive: 1. Socket alive: 1. Data available: 0. \ + # Configuration file change detected: 0. Requests received: 14. + + $stats{'client-requests-on-connection'}{$1}++; + $stats{'closed-client-connections'}++; } } -sub gather_loglevel_header_stats ($) { +sub gather_loglevel_header_stats($$) { - my $c = shift; - my $thread = shift; + my ($c, $thread) = @_; our %stats; + our %cli_options; if ($c =~ m/^A HTTP\/1\.1 response without/ or $c =~ m/^Keeping the server header 'Connection: keep-alive' around./) @@ -1950,57 +2258,180 @@ sub gather_loglevel_header_stats ($) { } } -sub init_stats () { +sub init_stats() { our %stats = ( requests => 0, + requests_clf => 0, crunches => 0, 'server-keep-alive' => 0, 'reused-connections' => 0, 'empty-responses' => 0, 'empty-responses-on-new-connections' => 0, 'empty-responses-on-reused-connections' => 0, + 'fast-redirections' => 0, + 'blocked' => 0, + 'connection-failure' => 0, + 'connection-timeout' => 0, + 'reused-connections' => 0, + 'server-keep-alive' => 0, + 'closed-client-connections' => 0, + 'content-size-total' => 0, ); + $stats{'client-requests-on-connection'}{1} = 0; } -sub get_percentage ($$) { +sub get_percentage($$) { my $big = shift; my $small = shift; + + # If small is 0 the percentage is always 0%. + # Make sure it works even if big is 0 as well. + return "0.00%" if ($small eq 0); + + # Prevent division by zero. + # XXX: Is this still supposed to be reachable? return "NaN" if ($big eq 0); + return sprintf("%.2f%%", $small / $big * 100); } -sub print_stats () { +sub print_stats() { our %stats; + our %cli_options; my $new_connections = $stats{requests} - $stats{crunches} - $stats{'reused-connections'}; - my $outgoing_requests = $stats{requests} - $stats{crunches}; + my $client_requests_checksum = 0; + my $requests_total; + + if ($stats{requests_clf} && $stats{requests} + && $stats{requests_clf} != $stats{requests}) { + print "Inconsistent request counts: " . $stats{requests} . "/" . $stats{requests_clf} . "\n"; + } + + # To get the total number of requests we can use either the number + # of Common-Log-Format lines or the number of "Request:" messages. + # We prefer the number of CLF lines if available because using + # it works when analysing old log files from Privoxy versions before 3.0.29. + # In Privoxy 3.0.28 and earlier "Request:" messages excluded + # crunched messages. + $requests_total = $stats{requests_clf} ? $stats{requests_clf} : $stats{requests}; - if ($stats{requests} eq 0) { + if ($requests_total eq 0) { print "No requests yet.\n"; return; } - print "Client requests total: " . $stats{requests} . "\n"; - print "Crunches: " . $stats{crunches} . " (" . - get_percentage($stats{requests}, $stats{crunches}) . ")\n"; - print "Outgoing requests: " . $outgoing_requests . " (" . - get_percentage($stats{requests}, $outgoing_requests) . ")\n"; + print "Client requests total: " . $requests_total . "\n"; + if ($stats{crunches}) { + my $outgoing_requests = $requests_total - $stats{crunches}; + print "Crunches: " . $stats{crunches} . " (" . + get_percentage($requests_total, $stats{crunches}) . ")\n"; + print "Blocks: " . $stats{'blocked'} . " (" . + get_percentage($requests_total, $stats{'blocked'}) . ")\n"; + print "Fast redirections: " . $stats{'fast-redirections'} . " (" . + get_percentage($requests_total, $stats{'fast-redirections'}) . ")\n"; + print "Connection timeouts: " . $stats{'connection-timeout'} . " (" . + get_percentage($requests_total, $stats{'connection-timeout'}) . ")\n"; + print "Connection failures: " . $stats{'connection-failure'} . " (" . + get_percentage($requests_total, $stats{'connection-failure'}) . ")\n"; + print "Outgoing requests: " . $outgoing_requests . " (" . + get_percentage($requests_total, $outgoing_requests) . ")\n"; + } else { + print "No crunches detected. Is 'debug 1024' enabled?\n"; + } + print "Server keep-alive offers: " . $stats{'server-keep-alive'} . " (" . - get_percentage($stats{requests}, $stats{'server-keep-alive'}) . ")\n"; + get_percentage($requests_total, $stats{'server-keep-alive'}) . ")\n"; print "New outgoing connections: " . $new_connections . " (" . - get_percentage($stats{requests}, $new_connections) . ")\n"; + get_percentage($requests_total, $new_connections) . ")\n"; print "Reused connections: " . $stats{'reused-connections'} . " (" . - get_percentage($stats{requests}, $stats{'reused-connections'}) . ")\n"; + get_percentage($requests_total, $stats{'reused-connections'}) . + "; server offers accepted: " . + get_percentage($stats{'server-keep-alive'}, $stats{'reused-connections'}) . ")\n"; print "Empty responses: " . $stats{'empty-responses'} . " (" . - get_percentage($stats{requests}, $stats{'empty-responses'}) . ")\n"; + get_percentage($requests_total, $stats{'empty-responses'}) . ")\n"; print "Empty responses on new connections: " . $stats{'empty-responses-on-new-connections'} . " (" . - get_percentage($stats{requests}, $stats{'empty-responses-on-new-connections'}) + get_percentage($requests_total, $stats{'empty-responses-on-new-connections'}) . ")\n"; print "Empty responses on reused connections: " . $stats{'empty-responses-on-reused-connections'} . " (" . - get_percentage($stats{requests}, $stats{'empty-responses-on-reused-connections'}) . + get_percentage($requests_total, $stats{'empty-responses-on-reused-connections'}) . ")\n"; + print "Client connections: " . $stats{'closed-client-connections'} . "\n"; + if ($stats{'content-size-total'}) { + print "Bytes of content transfered to the client: " . $stats{'content-size-total'} . "\n"; + } + my $lines_printed = 0; + print "Client requests per connection distribution:\n"; + foreach my $client_requests (sort { + $stats{'client-requests-on-connection'}{$b} <=> $stats{'client-requests-on-connection'}{$a}} + keys %{$stats{'client-requests-on-connection'} + }) + { + my $count = $stats{'client-requests-on-connection'}{$client_requests}; + $client_requests_checksum += $count * $client_requests; + if ($cli_options{'show-complete-request-distribution'} or ($lines_printed < 10)) { + printf "%8d: %d\n", $count, $client_requests; + $lines_printed++; + } + } + unless ($cli_options{'show-complete-request-distribution'}) { + printf "Enable --show-complete-request-distribution to get less common numbers as well.\n"; + } + # Due to log rotation we may not have a complete picture for all the requests + printf "Improperly accounted requests: ~%d\n", abs($requests_total - $client_requests_checksum); + + if (exists $stats{method}) { + print "Method distribution:\n"; + foreach my $method (sort {$stats{'method'}{$b} <=> $stats{'method'}{$a}} keys %{$stats{'method'}}) { + printf "%8d : %-8s\n", $stats{'method'}{$method}, $method; + } + } else { + print "Method distribution unknown. No CLF message parsed yet. Is 'debug 512' enabled?\n"; + } + if (exists $stats{'http-version'}) { + print "Client HTTP versions:\n"; + foreach my $http_version (sort {$stats{'http-version'}{$b} <=> $stats{'http-version'}{$a}} keys %{$stats{'http-version'}}) { + printf "%8d : %-8s\n", $stats{'http-version'}{$http_version}, $http_version; + } + } else { + print "HTTP version distribution unknown. No CLF message parsed yet. Is 'debug 512' enabled?\n"; + } + if (exists $stats{'status-code'}) { + print "HTTP status codes:\n"; + foreach my $status_code (sort {$stats{'status-code'}{$b} <=> $stats{'status-code'}{$a}} keys %{$stats{'status-code'}}) { + printf "%8d : %-8d\n", $stats{'status-code'}{$status_code}, $status_code; + } + } else { + print "Status code distribution unknown. No CLF message parsed yet. Is 'debug 512' enabled?\n"; + } + + if ($cli_options{'url-statistics-threshold'} == 0) { + print "URL statistics are disabled. Increase --url-statistics-threshold to enable them.\n"; + } else { + print "Requested URLs:\n"; + foreach my $resource (sort {$stats{'resource'}{$b} <=> $stats{'resource'}{$a}} keys %{$stats{'resource'}}) { + if ($stats{'resource'}{$resource} < $cli_options{'url-statistics-threshold'}) { + print "Skipped statistics for URLs below the treshold.\n"; + last; + } + printf "%d : %s\n", $stats{'resource'}{$resource}, $resource; + } + } + + if ($cli_options{'host-statistics-threshold'} == 0) { + print "Host statistics are disabled. Increase --host-statistics-threshold to enable them.\n"; + } else { + print "Requested Hosts:\n"; + foreach my $host (sort {$stats{'hosts'}{$b} <=> $stats{'hosts'}{$a}} keys %{$stats{'hosts'}}) { + if ($stats{'hosts'}{$host} < $cli_options{'host-statistics-threshold'}) { + print "Skipped statistics for Hosts below the treshold.\n"; + last; + } + printf "%d : %s\n", $stats{'hosts'}{$host}, $host; + } + } } @@ -2008,10 +2439,9 @@ sub print_stats () { # Functions that actually print stuff ################################################################################ -sub print_clf_message () { +sub print_clf_message() { our ($ip, $timestamp, $request_line, $status_code, $size); - our %h; my $output = ''; return if DEBUG_SUPPRESS_LOG_MESSAGES; @@ -2026,67 +2456,61 @@ sub print_clf_message () { $output .= $h{'Status'} . $status_code . $h{'Standard'}; $output .= " "; $output .= $h{'Number'} . $size . $h{'Standard'}; - $output .= get_line_end(); + $output .= $line_end; print $output; } -sub print_non_clf_message ($) { +sub print_non_clf_message($) { - our %req; - our %thread_colours; - our %h; - our $t; - our $time_colour_index; - our @time_colours; - my $output; my $content = shift; - my ($day, $time_stamp, $msecs, $thread, $log_level) - = ($req{$t}{'day'}, $req{$t}{'time-stamp'}, $req{$t}{'msecs'}, $t, $req{$t}{'log-level'} ); + my $date_string = $keep_date_mode ? $req{$t}{'day'} . ' ' : ''; + my $msec_string = $no_msecs_mode ? '' : '.' . $req{$t}{'msecs'}; + my $line_start = $html_output_mode ? '' : $h{"Standard"}; return if DEBUG_SUPPRESS_LOG_MESSAGES; - $output .= $h{"Standard"} unless cli_option_is_set('html-output'); - # $output .= "$day "; - $output .= $time_colours[$time_colour_index % 2]; + print $line_start + . $date_string + . $time_colours[$time_colour_index % 2] + . $req{$t}{'time-stamp'} + . $msec_string + . $h{Standard} . " " + . $thread_colours{$t} + . $t + . $h{Standard} + . " " + . $h{$req{$t}{'log-level'}} + . $req{$t}{'log-level'} + . $h{Standard} + . ": " + . $content + . $line_end; +} - $output .= $time_stamp; - $output .= ".$msecs" unless cli_option_is_set('no-msecs'); - $output .= $h{"Standard"}; - $output .= " "; - $output .= $thread_colours{$thread} if (defined($thread_colours{$thread})); - $output .= $thread; - $output .= $h{"Standard"} . " "; - $output .= $h{$log_level} if (defined($h{$log_level})); - $output .= $log_level; - $output .= $h{"Standard"} . ": "; - $output .= "$content"; - $output .= get_line_end(); +sub shorten_thread_id($) { - print $output; -} + my $thread_id = shift; -sub parse_loop () { + our %short_thread_ids; + our $max_threadid; - our $t; - our %req; # request data from previous lines - our %h; - our %thread_colours; - our @all_colours; - our @time_colours; - our $thread_colour_index = 0; - our $header_colour_index = 0; - our $time_colour_index = 0; + unless (defined $short_thread_ids{$thread_id}) { + $short_thread_ids{$thread_id} = sprintf "%.3d", $max_threadid++; + } + + return $short_thread_ids{$thread_id} +} + +sub parse_loop() { my ($day, $time_stamp, $thread, $log_level, $content, $c, $msecs); my $last_msecs = 0; my $last_thread = 0; my $last_timestamp = 0; - my $output; my $filters_that_did_nothing; my $key; my $time_colour; - our $no_special_header_highlighting; $time_colour = paint_it('white'); my %log_level_handlers = ( @@ -2103,27 +2527,21 @@ sub parse_loop () { 'Error' => \&handle_loglevel_error, 'Fatal error' => \&handle_loglevel_ignore, 'Writing' => \&handle_loglevel_ignore, + 'Received' => \&handle_loglevel_ignore, + 'Tagging' => \&handle_loglevel_tagging, + 'Actions' => \&handle_loglevel_ignore, 'Unknown log level' => \&handle_loglevel_ignore, ); while (<>) { - - $output = ''; - - if (m/^(\w{3} \d{2}) (\d\d:\d\d:\d\d)\.?(\d+)? (?:Privoxy\()?([^\)\s]*)[\)]? ([\w -]*): (.*?)\r?$/) { - # XXX: Put in req hash? - $day = $1; - $time_stamp = $2; - $msecs = $3 ? $3 : 0; # Only the cool kids have micro second resolution - $log_level = $5; - $content = $c = $6; - $thread = $t = $4; - - $req{$t}{'day'} = $day; - $req{$t}{'time-stamp'} = $time_stamp; - $req{$t}{'msecs'} = $msecs; # Only the cool kids have micro second resolution; - $req{$t}{'log-level'} = $log_level; - $req{$t}{'content'} = $content; + + if (m/^(\d{4}-\d{2}-\d{2}|\w{3} \d{2}) (\d\d:\d\d:\d\d)\.?(\d+)? (?:Privoxy\()?([^\)\s]*)[\)]? ([\w -]*): (.*?)\r?$/) { + $thread = $t = ($shorten_thread_ids) ? shorten_thread_id($4) : $4; + $req{$t}{'day'} = $day = $1; + $req{$t}{'time-stamp'} = $time_stamp = $2; + $req{$t}{'msecs'} = $msecs = $3 ? $3 : 0; # Only the cool kids have micro second resolution; + $req{$t}{'log-level'} = $log_level = $5; + $req{$t}{'content'} = $content = $c = $6; $req{$t}{'log-message'} = $_; $no_special_header_highlighting = 0; @@ -2134,12 +2552,11 @@ sub parse_loop () { } else { die "No handler found for log level \"$log_level\"\n"; - } - # Highlight Truncations - if (m/\.\.\. \[(too long, truncated)/) { - $content =~ s@($1)@$h{'Truncation'}$1$h{'Standard'}@g; + # Highlight Truncations + if (length($_) > 4000) { + $content =~ s@(too long, truncated)]$@$h{'Truncation'}$1$h{'Standard'}]@g; } next unless $content; @@ -2151,15 +2568,15 @@ sub parse_loop () { } # Switch timestamp colour if timestamps differ - if ($msecs != $last_msecs || !($time_stamp =~ m/$last_timestamp/)) { + if (($msecs ne $last_msecs) || ($time_stamp ne $last_timestamp)) { debug_message("Tick tack!") if DEBUG_TICKS; - $time_colour = $time_colours[$time_colour_index % 2]; - $time_colour_index++ + $time_colour = $time_colours[$time_colour_index % 2]; + $time_colour_index++; + $last_msecs = $msecs; + $last_timestamp = $time_stamp; } - $last_msecs = $msecs; $last_thread = $thread; - $last_timestamp = $time_stamp; print_non_clf_message($content); @@ -2172,7 +2589,7 @@ sub parse_loop () { our ($ip, $timestamp, $request_line, $status_code, $size) = ($1, $2, $3, $4, $5); print_clf_message(); - + } else { # Some Privoxy log messages span more than one line, @@ -2186,44 +2603,49 @@ sub parse_loop () { } } -sub stats_loop () { +sub stats_loop() { - my ($day, $time_stamp, $thread, $log_level, $content, $c, $msecs); + my ($day, $time_stamp, $msecs, $thread, $log_level, $content); + my $strict_checks = cli_option_is_set('strict-checks'); my %log_level_handlers = ( - 'Re-Filter' => \&handle_loglevel_ignore, - 'Header' => \&gather_loglevel_header_stats, - 'Connect' => \&gather_loglevel_connect_stats, - 'Redirect' => \&handle_loglevel_ignore, - 'Request' => \&gather_loglevel_request_stats, - 'Crunch' => \&gather_loglevel_crunch_stats, - 'Gif-Deanimate' => \&handle_loglevel_ignore, - 'Info' => \&handle_loglevel_ignore, - 'CGI' => \&handle_loglevel_ignore, - 'Force' => \&handle_loglevel_ignore, - 'Error' => \&gather_loglevel_error_stats, - 'Fatal error' => \&handle_loglevel_ignore, - 'Writing' => \&handle_loglevel_ignore, - 'Unknown log level' => \&handle_loglevel_ignore + 'Connect:' => \&gather_loglevel_connect_stats, + 'Crunch:' => \&gather_loglevel_crunch_stats, + 'Error:' => \&gather_loglevel_error_stats, + 'Header:' => \&gather_loglevel_header_stats, + 'Request:' => \&gather_loglevel_request_stats, + ); + my %ignored_log_levels = ( + 'Actions:' => \&handle_loglevel_ignore, + 'CGI:' => \&handle_loglevel_ignore, + 'Fatal error:' => \&handle_loglevel_ignore, + 'Force:' => \&handle_loglevel_ignore, + 'Gif-Deanimate:' => \&handle_loglevel_ignore, + 'Info:' => \&handle_loglevel_ignore, + 'Re-Filter:' => \&handle_loglevel_ignore, + 'Received:' => \&handle_loglevel_ignore, + 'Redirect:' => \&handle_loglevel_ignore, + 'Unknown log level:' => \&handle_loglevel_ignore, + 'Writing:' => \&handle_loglevel_ignore, + 'Tagging:' => \&handle_loglevel_ignore, ); while (<>) { - if (m/^(\w{3} \d{2}) (\d\d:\d\d:\d\d)\.?(\d+)? (?:Privoxy\()?([^\)\s]*)[\)]? ([\w -]*): (.*?)\r?$/) { - $day = $1; - $time_stamp = $2; - $msecs = $3 ? $3 : 0; - $log_level = $5; - $content = $c = $6; - $thread = $4; + (undef, $time_stamp, $thread, $log_level, $content) = split(/ /, $_, 5); - if (defined($log_level_handlers{$log_level})) { - $content = $log_level_handlers{$log_level}($content, $thread); + next if (not defined($log_level)); - } else { + if ($time_stamp eq "-") { - die "No handler found for log level \"$log_level\"\n"; + gather_loglevel_clf_stats($content); - } + } elsif (defined($log_level_handlers{$log_level})) { + + $content = $log_level_handlers{$log_level}($content, $thread); + + } elsif ($strict_checks and not defined($ignored_log_levels{$log_level})) { + + die "No handler found for: $_"; } } @@ -2231,44 +2653,85 @@ sub stats_loop () { } +sub unbreak_lines_only_loop() { + my $log_messages_reached = 0; + while (<>) { + chomp; + + # Log level other than LOG_LEVEL_CLF? + if (m/^(\d{4}-\d{2}-\d{2}|\w{3} \d{2}) (\d\d:\d\d:\d\d)\.?(\d+)? (?:Privoxy\()?([^\)\s]*)[\)]? ([\w -]*): (.*?)\r?$/ or + # LOG_LEVEL_CLF? + m/^((?:\d+\.\d+\.\d+\.\d+)) - - \[(.*)\] "(.*)" (\d+) (\d+)/) { + $log_messages_reached = 1; + print "\n"; + + } else { + # Wrapped message + $_ = "\n". $_ if /^(?:\d+\.\d+\.\d+\.\d+)/; + $_ = " " . $_; + } + s@
$@@; + print; + print "\n" unless $log_messages_reached; + } + print "\n"; +} + sub VersionMessage { my $version_message; $version_message .= 'Privoxy-Log-Parser ' . PRIVOXY_LOG_PARSER_VERSION . "\n"; - $version_message .= 'Copyright (C) 2007-2009 Fabian Keil ' . "\n"; - $version_message .= 'http://www.fabiankeil.de/sourcecode/privoxy-log-parser/' . "\n"; + $version_message .= 'https://www.fabiankeil.de/sourcecode/privoxy-log-parser/' . "\n"; print $version_message; } -sub get_cli_options () { +sub get_cli_options() { our %cli_options = ( 'html-output' => CLI_OPTION_DEFAULT_TO_HTML_OUTPUT, 'title' => CLI_OPTION_TITLE, + 'keep-date' => CLI_OPTION_KEEP_DATE, 'no-syntax-highlighting' => CLI_OPTION_NO_SYNTAX_HIGHLIGHTING, 'no-embedded-css' => CLI_OPTION_NO_EMBEDDED_CSS, 'no-msecs' => CLI_OPTION_NO_MSECS, + 'shorten-thread-ids' => CLI_OPTION_SHORTEN_THREAD_IDS, 'show-ineffective-filters' => CLI_OPTION_SHOW_INEFFECTIVE_FILTERS, - 'accept-unknown-messages' => CLI_OPTION_ACCEPT_UNKNOWN_MESSAGES, 'statistics' => CLI_OPTION_STATISTICS, - ); + 'strict-checks' => CLI_OPTION_STRICT_CHECKS, + 'url-statistics-threshold' => CLI_OPTION_URL_STATISTICS_THRESHOLD, + 'unbreak-lines-only' => CLI_OPTION_UNBREAK_LINES_ONLY, + 'host-statistics-threshold'=> CLI_OPTION_HOST_STATISTICS_THRESHOLD, + 'show-complete-request-distribution' => CLI_OPTION_SHOW_COMPLETE_REQUEST_DISTRIBUTION, + ); GetOptions ( 'html-output' => \$cli_options{'html-output'}, 'title' => \$cli_options{'title'}, + 'keep-date' => \$cli_options{'keep-date'}, 'no-syntax-highlighting' => \$cli_options{'no-syntax-highlighting'}, 'no-embedded-css' => \$cli_options{'no-embedded-css'}, 'no-msecs' => \$cli_options{'no-msecs'}, + 'shorten-thread-ids' => \$cli_options{'shorten-thread-ids'}, 'show-ineffective-filters' => \$cli_options{'show-ineffective-filters'}, - 'accept-unknown-messages' => \$cli_options{'accept-unknown-messages'}, 'statistics' => \$cli_options{'statistics'}, + 'strict-checks' => \$cli_options{'strict-checks'}, + 'unbreak-lines-only' => \$cli_options{'unbreak-lines-only'}, + 'url-statistics-threshold=i'=> \$cli_options{'url-statistics-threshold'}, + 'host-statistics-threshold=i'=> \$cli_options{'host-statistics-threshold'}, + 'show-complete-request-distribution' => \$cli_options{'show-complete-request-distribution'}, 'version' => sub { VersionMessage && exit(0) }, 'help' => \&help, - ); + ) or exit(1); + + $html_output_mode = cli_option_is_set('html-output'); + $no_msecs_mode = cli_option_is_set('no-msecs'); + $keep_date_mode = cli_option_is_set('keep-date'); + $shorten_thread_ids = cli_option_is_set('shorten-thread-ids'); + $line_end = get_line_end(); } -sub help () { +sub help() { our %cli_options; @@ -2277,13 +2740,17 @@ sub help () { print << " EOF" Options and their default values if they have any: - [--accept-unknown-messages] + [--host-statistics-threshold $cli_options{'host-statistics-threshold'}] [--html-output] [--no-embedded-css] [--no-msecs] [--no-syntax-highlighting] + [--shorten-thread-ids] [--show-ineffective-filters] + [--show-complete-request-distribution] [--statistics] + [--unbreak-lines-only] + [--url-statistics-threshold $cli_options{'url-statistics-threshold'}] [--title $cli_options{'title'}] [--version] see "perldoc $0" for more information @@ -2295,7 +2762,7 @@ see "perldoc $0" for more information ################################################################################ # main ################################################################################ -sub main () { +sub main() { get_cli_options(); set_background(DEFAULT_BACKGROUND); @@ -2303,7 +2770,10 @@ sub main () { print_intro(); - if (cli_option_is_set('statistics')) { + # XXX: should explicitly reject incompatible argument combinations + if (cli_option_is_set('unbreak-lines-only')) { + unbreak_lines_only_loop(); + } elsif (cli_option_is_set('statistics')) { stats_loop(); } else { parse_loop(); @@ -2320,9 +2790,10 @@ B - A parser and syntax-highlighter for Privoxy log messages =head1 SYNOPSIS -B [B<--accept-unknown-messages>] [B<--html-output>] -[B<--no-msecs>] [B<--no-syntax-higlighting>] [B<--show-ineffective-filters>] -[B<--version>] +B [B<--html-output>] +[B<--no-msecs>] [B<--no-syntax-higlighting>] [B<--statistics>] +[B<--shorten-thread-ids>] [B<--show-ineffective-filters>] +[B<--url-statistics-threshold>] [B<--version>] =head1 DESCRIPTION @@ -2337,7 +2808,7 @@ B reads Privoxy log messages and - (in some cases) calculates additional information, like the compression ratio or how a filter affected the content size. - + With B you should be able to increase Privoxy's log level without getting confused by the resulting amount of output. For example for "debug 64" B will (by default) only show messages that @@ -2346,29 +2817,58 @@ will hide the "filter foo caused 0 hits" message. =head1 OPTIONS -[B<--accept-unknown-messages>] Don't print warnings in case of unknown messages, -just don't highlight them. +[B<--host-statistics-threshold>] Only show the request count for a host +if it's above or equal to the given threshold. If the threshold is 0, host +statistics are disabled. [B<--html-output>] Use HTML and CSS for the syntax highlighting. If this option is omitted, ANSI escape sequences are used unless B<--no-syntax-highlighting> is active. This option is only intended to make embedding log excerpts in web pages easier. It does not escape any input! -[B<--no-msecs>] Don't expect milisecond resolution +[B<--keep-date>] Don't remove the date when printing highlighted log messages. +Useful when parsing multiple log files at once. + +[B<--no-msecs>] Don't expect millisecond resolution [B<--no-syntax-highlighting>] Disable syntax-highlighting. Useful when the filtered output is piped into less in which case the ANSI control codes don't work, or if the terminal itself doesn't support the control codes. +[B<--shorten-thread-ids>] Shorten the thread ids to a three-digit decimal number. +Note that the mapping from thread ids to shortened ids is created at run-time +and thus varies with the input. + [B<--show-ineffective-filters>] Don't suppress log lines for filters that didn't modify the content. +[B<--show-complete-request-distribution>] Show the complete client request +distribution in the B<--statistics> output. Without this option only the +ten most common numbers are shown. + [B<--statistics>] Gather various statistics instead of syntax highlighting log messages. This is an experimental feature, if the results look wrong -they very well might be. Also note that the results a pretty much guaranteed +they very well might be. Also note that the results are pretty much guaranteed to be incorrect if Privoxy and Privoxy-Log-Parser aren't in sync. +[B<--strict-checks>] When generating statistics, look more careful at the +input data and abort if it is unexpected, even if it doesn't affect the +results. Significantly slows the parsing down and is not expected to catch +any problems that matter. +When highlighting, print warnings in case of unknown messages which can't be +properly highlighted. + +[B<--unbreak-lines-only>] Tries to fix lines that got messed up by a broken or +interestingly configured mail client and thus are no longer recognized properly. +Only fixes some breakage, but may be good enough or at least better than nothing. +Doesn't do anything else, so you probably want to pipe the output into +B again. + +[B<--url-statistics-threshold>] Only show the request count for a resource +if it's above or equal to the given threshold. If the threshold is 0, URL +statistics are disabled. + [B<--version>] Print version and exit. =head1 EXAMPLES @@ -2418,7 +2918,7 @@ Many settings can't be controlled through command line options yet. =head1 SEE ALSO -privoxy(1) +privoxy(8) =head1 AUTHOR