Bump copyright year.

[privoxy.git] / tools / privoxy-log-parser.pl

diff --git a/tools/privoxy-log-parser.pl b/tools/privoxy-log-parser.pl
index 46271f2..1b351bb 100755 (executable)
--- a/tools/privoxy-log-parser.pl
+++ b/tools/privoxy-log-parser.pl
@@ -8,7 +8,7 @@
 #
 # http://www.fabiankeil.de/sourcecode/privoxy-log-parser/
 #
 #
 # http://www.fabiankeil.de/sourcecode/privoxy-log-parser/
 #

-# $Id: privoxy-log-parser.pl,v 1.76 2010/01/11 11:48:23 fabiankeil Exp $
+# $Id: privoxy-log-parser.pl,v 1.83 2010/07/22 14:51:59 fabiankeil Exp $

 #
 # TODO:
 #       - LOG_LEVEL_CGI, LOG_LEVEL_ERROR, LOG_LEVEL_WRITE content highlighting
 #
 # TODO:
 #       - LOG_LEVEL_CGI, LOG_LEVEL_ERROR, LOG_LEVEL_WRITE content highlighting


@@ -24,7 +24,7 @@
 #       - Use generic highlighting function that takes a regex and the
 #         hash key as input.
 #
 #       - Use generic highlighting function that takes a regex and the
 #         hash key as input.
 #

-# Copyright (c) 2007-2009 Fabian Keil <fk@fabiankeil.de>
+# Copyright (c) 2007-2010 Fabian Keil <fk@fabiankeil.de>

 #
 # Permission to use, copy, modify, and distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
 #
 # Permission to use, copy, modify, and distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above


@@ -56,6 +56,7 @@ use constant {
     CLI_OPTION_NO_EMBEDDED_CSS => 0,
     CLI_OPTION_NO_MSECS => 0,
     CLI_OPTION_NO_SYNTAX_HIGHLIGHTING => 0,
     CLI_OPTION_NO_EMBEDDED_CSS => 0,
     CLI_OPTION_NO_MSECS => 0,
     CLI_OPTION_NO_SYNTAX_HIGHLIGHTING => 0,

+    CLI_OPTION_SHORTEN_THREAD_IDS => 0,

     CLI_OPTION_SHOW_INEFFECTIVE_FILTERS => 0,
     CLI_OPTION_ACCEPT_UNKNOWN_MESSAGES => 0,
     CLI_OPTION_STATISTICS => 0,
     CLI_OPTION_SHOW_INEFFECTIVE_FILTERS => 0,
     CLI_OPTION_ACCEPT_UNKNOWN_MESSAGES => 0,
     CLI_OPTION_STATISTICS => 0,


@@ -105,6 +106,7 @@ my $header_highlight_regex = '';
 
 my $html_output_mode;
 my $no_msecs_mode; # XXX: should probably be removed
 
 my $html_output_mode;
 my $no_msecs_mode; # XXX: should probably be removed

+my $shorten_thread_ids;

 my $line_end;
 
 sub prepare_our_stuff () {
 my $line_end;
 
 sub prepare_our_stuff () {


@@ -908,6 +910,7 @@ sub handle_loglevel_header ($) {
           or $c =~ m/^Appended client IP address to/
           or $c =~ m/^Removing 'Connection: close' to imply keep-alive./
           or $c =~ m/^keep-alive support is disabled/
           or $c =~ m/^Appended client IP address to/
           or $c =~ m/^Removing 'Connection: close' to imply keep-alive./
           or $c =~ m/^keep-alive support is disabled/

+          or $c =~ m/^Continue hack in da house/

             )
     {
         # XXX: Some of these may need highlighting
             )
     {
         # XXX: Some of these may need highlighting


@@ -953,6 +956,7 @@ sub handle_loglevel_header ($) {
         # Appended client IP address to X-Forwarded-For: 10.0.0.2, 10.0.0.1
         # Removing 'Connection: close' to imply keep-alive.
         # keep-alive support is disabled. Crunching: Keep-Alive: 300.
         # Appended client IP address to X-Forwarded-For: 10.0.0.2, 10.0.0.1
         # Removing 'Connection: close' to imply keep-alive.
         # keep-alive support is disabled. Crunching: Keep-Alive: 300.

+        # Continue hack in da house.

 
     } elsif ($c =~ m/^scanning headers for:/) {
 
 
     } elsif ($c =~ m/^scanning headers for:/) {
 


@@ -1622,7 +1626,7 @@ sub handle_loglevel_connect ($) {
         # Waiting for up to 4999 bytes from the client.
         $c =~ s@(?<=up to )(\d+)@$h{'Number'}$1$h{'Standard'}@;
 
         # Waiting for up to 4999 bytes from the client.
         $c =~ s@(?<=up to )(\d+)@$h{'Number'}$1$h{'Standard'}@;
 

-    } elsif ($c =~ m/^Looks like we rea/ or
+    } elsif ($c =~ m/^Looks like we / or

              $c =~ m/^Unsetting keep-alive flag/ or
              $c =~ m/^No connections to wait/ or
              $c =~ m/^Complete client request received/ or
              $c =~ m/^Unsetting keep-alive flag/ or
              $c =~ m/^No connections to wait/ or
              $c =~ m/^Complete client request received/ or


@@ -1637,6 +1641,8 @@ sub handle_loglevel_connect ($) {
         # Looks like we reached the end of the last chunk. We better stop reading.
         # Looks like we read the end of the last chunk together with the server \
         #  headers. We better stop reading.
         # Looks like we reached the end of the last chunk. We better stop reading.
         # Looks like we read the end of the last chunk together with the server \
         #  headers. We better stop reading.

+        # Looks like we got the last chunk together with the server headers. \
+        #  We better stop reading.

         # Unsetting keep-alive flag.
         # No connections to wait for left.
         # Client request arrived in time or the client closed the connection.
         # Unsetting keep-alive flag.
         # No connections to wait for left.
         # Client request arrived in time or the client closed the connection.


@@ -1687,6 +1693,12 @@ sub handle_loglevel_info ($) {
         # Reloading configuration file '/usr/local/etc/privoxy/config'
         $c =~ s@(?<=loading configuration file \')([^\']*)@$h{'file'}$1$h{'Standard'}@;
 
         # Reloading configuration file '/usr/local/etc/privoxy/config'
         $c =~ s@(?<=loading configuration file \')([^\']*)@$h{'file'}$1$h{'Standard'}@;
 

+    } elsif ($c =~ m/^Loading (actions|filter) file: /) {
+
+        # Loading actions file: /usr/local/etc/privoxy/default.action
+        # Loading filter file: /usr/local/etc/privoxy/default.filter
+        $c =~ s@(?<= file: )(.*)$@$h{'file'}$1$h{'Standard'}@;
+

     } elsif ($c =~ m/^exiting by signal/) {
 
         # exiting by signal 15 .. bye
     } elsif ($c =~ m/^exiting by signal/) {
 
         # exiting by signal 15 .. bye


@@ -1744,7 +1756,8 @@ sub handle_loglevel_info ($) {
              $c =~ m/^Malformerd HTTP headers detected and MS IIS5 hack enabled/ or
              $c =~ m/^Invalid \"chunked\" transfer/ or
              $c =~ m/^Support for/ or
              $c =~ m/^Malformerd HTTP headers detected and MS IIS5 hack enabled/ or
              $c =~ m/^Invalid \"chunked\" transfer/ or
              $c =~ m/^Support for/ or

-             $c =~ m/^Flushing header and buffers/
+             $c =~ m/^Flushing header and buffers/ or
+             $c =~ m/^Can not resolve/

              ) {
 
         # No logfile configured. Please enable it before reporting any problems.
              ) {
 
         # No logfile configured. Please enable it before reporting any problems.


@@ -1755,6 +1768,7 @@ sub handle_loglevel_info ($) {
         # Support for 'Connection: keep-alive' is experimental, incomplete and\
         #  known not to work properly in some situations.
         # Flushing header and buffers. Stepping back from filtering.
         # Support for 'Connection: keep-alive' is experimental, incomplete and\
         #  known not to work properly in some situations.
         # Flushing header and buffers. Stepping back from filtering.

+        # Can not resolve doesnotexist: hostname nor servname provided, or not known

 
     } else {
 
 
     } else {
 


@@ -2023,6 +2037,20 @@ sub print_non_clf_message ($) {
         . $line_end;
 }
 
         . $line_end;
 }
 

+sub shorten_thread_id ($) {
+
+    my $thread_id = shift;
+
+    our %short_thread_ids;
+    our $max_threadid;
+
+    unless (defined $short_thread_ids{$thread_id}) {
+        $short_thread_ids{$thread_id} = sprintf "%.3d", $max_threadid++;
+    }
+
+    return $short_thread_ids{$thread_id}
+}
+

 sub parse_loop () {
 
     my ($day, $time_stamp, $thread, $log_level, $content, $c, $msecs);
 sub parse_loop () {
 
     my ($day, $time_stamp, $thread, $log_level, $content, $c, $msecs);


@@ -2054,7 +2082,7 @@ sub parse_loop () {
     while (<>) {
 
         if (m/^(\w{3} \d{2}) (\d\d:\d\d:\d\d)\.?(\d+)? (?:Privoxy\()?([^\)\s]*)[\)]? ([\w -]*): (.*?)\r?$/) {
     while (<>) {
 
         if (m/^(\w{3} \d{2}) (\d\d:\d\d:\d\d)\.?(\d+)? (?:Privoxy\()?([^\)\s]*)[\)]? ([\w -]*): (.*?)\r?$/) {

-            $thread = $t = $4;
+            $thread = $t = ($shorten_thread_ids) ? shorten_thread_id($4) : $4;

             $req{$t}{'day'} = $day = $1;
             $req{$t}{'time-stamp'} = $time_stamp = $2;
             $req{$t}{'msecs'} = $msecs = $3 ? $3 : 0; # Only the cool kids have micro second resolution;
             $req{$t}{'day'} = $day = $1;
             $req{$t}{'time-stamp'} = $time_stamp = $2;
             $req{$t}{'msecs'} = $msecs = $3 ? $3 : 0; # Only the cool kids have micro second resolution;


@@ -2170,7 +2198,7 @@ sub VersionMessage {
     my $version_message;
 
     $version_message .= 'Privoxy-Log-Parser ' . PRIVOXY_LOG_PARSER_VERSION  . "\n";
     my $version_message;
 
     $version_message .= 'Privoxy-Log-Parser ' . PRIVOXY_LOG_PARSER_VERSION  . "\n";

-    $version_message .= 'Copyright (C) 2007-2009 Fabian Keil <fk@fabiankeil.de>' . "\n";
+    $version_message .= 'Copyright (C) 2007-2010 Fabian Keil <fk@fabiankeil.de>' . "\n";

     $version_message .= 'http://www.fabiankeil.de/sourcecode/privoxy-log-parser/' . "\n";
 
     print $version_message;
     $version_message .= 'http://www.fabiankeil.de/sourcecode/privoxy-log-parser/' . "\n";
 
     print $version_message;


@@ -2184,6 +2212,7 @@ sub get_cli_options () {
         'no-syntax-highlighting'   => CLI_OPTION_NO_SYNTAX_HIGHLIGHTING,
         'no-embedded-css'          => CLI_OPTION_NO_EMBEDDED_CSS,
         'no-msecs'                 => CLI_OPTION_NO_MSECS,
         'no-syntax-highlighting'   => CLI_OPTION_NO_SYNTAX_HIGHLIGHTING,
         'no-embedded-css'          => CLI_OPTION_NO_EMBEDDED_CSS,
         'no-msecs'                 => CLI_OPTION_NO_MSECS,

+        'shorten-thread-ids'       => CLI_OPTION_SHORTEN_THREAD_IDS,

         'show-ineffective-filters' => CLI_OPTION_SHOW_INEFFECTIVE_FILTERS,
         'accept-unknown-messages'  => CLI_OPTION_ACCEPT_UNKNOWN_MESSAGES,
         'statistics'               => CLI_OPTION_STATISTICS,
         'show-ineffective-filters' => CLI_OPTION_SHOW_INEFFECTIVE_FILTERS,
         'accept-unknown-messages'  => CLI_OPTION_ACCEPT_UNKNOWN_MESSAGES,
         'statistics'               => CLI_OPTION_STATISTICS,


@@ -2195,6 +2224,7 @@ sub get_cli_options () {
         'no-syntax-highlighting'   => \$cli_options{'no-syntax-highlighting'},
         'no-embedded-css'          => \$cli_options{'no-embedded-css'},
         'no-msecs'                 => \$cli_options{'no-msecs'},
         'no-syntax-highlighting'   => \$cli_options{'no-syntax-highlighting'},
         'no-embedded-css'          => \$cli_options{'no-embedded-css'},
         'no-msecs'                 => \$cli_options{'no-msecs'},

+        'shorten-thread-ids'       => \$cli_options{'shorten-thread-ids'},

         'show-ineffective-filters' => \$cli_options{'show-ineffective-filters'},
         'accept-unknown-messages'  => \$cli_options{'accept-unknown-messages'},
         'statistics'               => \$cli_options{'statistics'},
         'show-ineffective-filters' => \$cli_options{'show-ineffective-filters'},
         'accept-unknown-messages'  => \$cli_options{'accept-unknown-messages'},
         'statistics'               => \$cli_options{'statistics'},


@@ -2204,6 +2234,7 @@ sub get_cli_options () {
 
    $html_output_mode = cli_option_is_set('html-output');
    $no_msecs_mode = cli_option_is_set('no-msecs');
 
    $html_output_mode = cli_option_is_set('html-output');
    $no_msecs_mode = cli_option_is_set('no-msecs');

+   $shorten_thread_ids = cli_option_is_set('shorten-thread-ids');

    $line_end = get_line_end();
 }
 
    $line_end = get_line_end();
 }
 


@@ -2221,6 +2252,7 @@ Options and their default values if they have any:
     [--no-embedded-css]
     [--no-msecs]
     [--no-syntax-highlighting]
     [--no-embedded-css]
     [--no-msecs]
     [--no-syntax-highlighting]

+    [--shorten-thread-ids]

     [--show-ineffective-filters]
     [--statistics]
     [--title $cli_options{'title'}]
     [--show-ineffective-filters]
     [--statistics]
     [--title $cli_options{'title'}]


@@ -2260,8 +2292,8 @@ B<privoxy-log-parser> - A parser and syntax-highlighter for Privoxy log messages
 =head1 SYNOPSIS
 
 B<privoxy-log-parser> [B<--accept-unknown-messages>] [B<--html-output>]
 =head1 SYNOPSIS
 
 B<privoxy-log-parser> [B<--accept-unknown-messages>] [B<--html-output>]

-[B<--no-msecs>] [B<--no-syntax-higlighting>] [B<--show-ineffective-filters>]
-[B<--version>]
+[B<--no-msecs>] [B<--no-syntax-higlighting>] [B<--statistics>]
+[B<--shorten-thread-ids>] [B<--show-ineffective-filters>] [B<--version>]

 
 =head1 DESCRIPTION
 
 
 =head1 DESCRIPTION
 


@@ -2300,6 +2332,10 @@ the filtered output is piped into less in which case the ANSI control
 codes don't work, or if the terminal itself doesn't support the control
 codes.
 
 codes don't work, or if the terminal itself doesn't support the control
 codes.
 

+[B<--shorten-thread-ids>] Shorten the thread ids to a three-digit decimal number.
+Note that the mapping from thread ids to shortended ids is created at
+run-time and thus varies with the input.
+

 [B<--show-ineffective-filters>] Don't suppress log lines for filters
 that didn't modify the content.
 
 [B<--show-ineffective-filters>] Don't suppress log lines for filters
 that didn't modify the content.