X-Git-Url: http://www.privoxy.org/gitweb/?p=privoxy.git;a=blobdiff_plain;f=tools%2Fprivoxy-regression-test.pl;h=c08cc76ad4f8a2b39c10a17a2e71b2fa52abe0ed;hp=22c9701de61c7a2a8a481fc6f5c4f0d64c07fce8;hb=29cb9a10e83d3e74040882af7c711ecd01c3f680;hpb=e57aed5199c1aebadda9ad6e02284e580867d696 diff --git a/tools/privoxy-regression-test.pl b/tools/privoxy-regression-test.pl index 22c9701d..c08cc76a 100755 --- a/tools/privoxy-regression-test.pl +++ b/tools/privoxy-regression-test.pl @@ -7,7 +7,7 @@ # A regression test "framework" for Privoxy. For documentation see: # perldoc privoxy-regression-test.pl # -# $Id: privoxy-regression-test.pl,v 1.81 2011/10/30 16:22:29 fabiankeil Exp $ +# $Id: privoxy-regression-test.pl,v 1.96 2016/05/12 08:42:57 fabiankeil Exp $ # # Wish list: # @@ -19,7 +19,7 @@ # - Document magic Expect Header values # - Internal fuzz support? # -# Copyright (c) 2007-2011 Fabian Keil +# Copyright (c) 2007-2016 Fabian Keil # # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above @@ -40,7 +40,7 @@ use strict; use Getopt::Long; use constant { - PRT_VERSION => 'Privoxy-Regression-Test 0.5', + PRT_VERSION => 'Privoxy-Regression-Test 0.7', CURL => 'curl', @@ -66,8 +66,6 @@ use constant { DEBUG_LEVEL_VERBOSE_SUCCESS => 0, DEBUG_LEVEL_STATUS => 1, - VERBOSE_TEST_DESCRIPTION => 1, - # Internal use, don't modify # Available debug bits: LL_SOFT_ERROR => 1, @@ -92,7 +90,6 @@ sub init_our_variables () { our $leading_log_time = LEADING_LOG_TIME; our $leading_log_date = LEADING_LOG_DATE; our $privoxy_cgi_url = PRIVOXY_CGI_URL; - our $verbose_test_description = VERBOSE_TEST_DESCRIPTION; our $log_level = get_default_log_level(); } @@ -311,6 +308,7 @@ sub load_regression_tests_through_privoxy () { my $curl_url = ''; my $file_number = 0; my $feature; + my $privoxy_version = '(Unknown version!)'; $curl_url .= $privoxy_cgi_url; $curl_url .= 'show-status'; @@ -342,10 +340,14 @@ sub load_regression_tests_through_privoxy () { $privoxy_features{$feature} = $1 if defined $feature; $feature = undef; + + } elsif (m@This is Privoxy (\d+\.\d+\.\d+) on@) { + $privoxy_version = $1; } } - l(LL_FILE_LOADING, "Recognized " . @actionfiles . " actions files"); + l(LL_STATUS, "Gathering regression tests from " . + @actionfiles . " action file(s) delivered by Privoxy $privoxy_version."); load_action_files(\@actionfiles); } @@ -474,6 +476,33 @@ sub enlist_new_test ($$$$$$) { "Regression test " . $number . " (section:" . $si . "):"); } +sub mark_matching_tests_for_skipping($) { + my $overwrite_condition = shift; + + our @regression_tests; + + for (my $s = 0; $s < @regression_tests; $s++) { + + my $r = 0; + + while (defined $regression_tests[$s][$r]) { + + if ($regression_tests[$s][$r]{'data'} eq $overwrite_condition) { + my $message = sprintf("Marking test %s for ignoring. Overwrite condition: %s.", + $regression_tests[$s][$r]{'number'}, $overwrite_condition); + + l(LL_FILE_LOADING, $message); + + # XXX: Should eventually get its own key so get_skip_reason() + # can tell about the overwrite condition. + $regression_tests[$s][$r]{'ignore'} = 1; + } + $r++; + } + } +} + + # XXX: Shares a lot of code with load_regression_tests_from_file() # that should be factored out. sub load_action_files ($) { @@ -491,14 +520,12 @@ sub load_action_files ($) { my $ignored = 0; - l(LL_STATUS, "Gathering regression tests from " . - @actionfiles . " action file(s) delivered by Privoxy."); - for my $file_number (0 .. @actionfiles - 1) { my $curl_url = quote($actionfiles[$file_number]); my $actionfile = undef; my $sticky_actions = undef; + my $level_offset = 0; foreach (@{get_cgi_page_or_else($curl_url)}) { @@ -518,6 +545,11 @@ sub load_action_files ($) { next unless defined $token; # Load regression tests + if ($token eq 'default level offset') { + + $level_offset = $value; + l(LL_FILE_LOADING, "Setting default level offset to " . $level_offset); + } if (token_starts_new_test($token)) { @@ -526,6 +558,9 @@ sub load_action_files ($) { $count++; enlist_new_test(\@regression_tests, $token, $value, $si, $ri, $count); $no_checks = 1; # Already validated by enlist_new_test(). + if ($level_offset != 0) { + $regression_tests[$si][$ri]{'level'} += $level_offset; + } } if ($token =~ /level\s+(\d+)/i) { @@ -543,7 +578,16 @@ sub load_action_files ($) { "action parameters are currently unsupported."); } } - + + if ($token eq 'overwrite condition') { + + l(LL_FILE_LOADING, "Detected overwrite condition: " . $value); + # We can only skip matching tests that have already + # be loaded but that is exactly what we want anyway. + mark_matching_tests_for_skipping($value); + next; + } + if ($si == -1 || $ri == -1) { # No beginning of a test detected yet, # so we don't care about any other test @@ -681,12 +725,12 @@ sub execute_regression_tests () { # from different sections isn't possible. # Is this worth changing the layout? fisher_yates_shuffle(\@regression_tests); - for (my $s = 0; $s < @regression_tests; $s++) { + for (my $s = 0; $s < @regression_tests; $s++) { fisher_yates_shuffle($regression_tests[$s]); } } - for (my $s = 0; $s < @regression_tests; $s++) { + for (my $s = 0; $s < @regression_tests; $s++) { my $r = 0; @@ -1507,86 +1551,81 @@ sub log_message ($) { sub log_result ($$) { - our $verbose_test_description; our $filtered_request; my $test = shift; my $result = shift; my $number = shift; - my $message = ''; + my $message = sprintf("%s for test %d", + interpret_result($result), + $test->{'number'}); - $message .= interpret_result($result); - $message .= " for test "; - $message .= $number; - $message .= '/'; - $message .= $test->{'number'}; - $message .= '/'; - $message .= $test->{'section-id'}; - $message .= '/'; - $message .= $test->{'regression-test-id'}; - $message .= '.'; + if (cli_option_is_set('verbose')) { + $message .= sprintf(" (%d/%d/%d)", $number, + $test->{'section-id'}, + $test->{'regression-test-id'}); + } - if ($verbose_test_description) { + $message .= '. '; - if ($test->{'type'} == CLIENT_HEADER_TEST) { + if ($test->{'type'} == CLIENT_HEADER_TEST) { - $message .= ' Header '; - $message .= quote($test->{'data'}); - $message .= ' and tag '; - $message .= quote($test->{'tag'}); + $message .= 'Header '; + $message .= quote($test->{'data'}); + $message .= ' and tag '; + $message .= quote($test->{'tag'}); - } elsif ($test->{'type'} == SERVER_HEADER_TEST) { + } elsif ($test->{'type'} == SERVER_HEADER_TEST) { - $message .= ' Request Header '; - $message .= quote($test->{'data'}); - $message .= ' and tag '; - $message .= quote($test->{'tag'}); + $message .= 'Request Header '; + $message .= quote($test->{'data'}); + $message .= ' and tag '; + $message .= quote($test->{'tag'}); - } elsif ($test->{'type'} == DUMB_FETCH_TEST) { + } elsif ($test->{'type'} == DUMB_FETCH_TEST) { - $message .= ' URL '; - $message .= quote($test->{'data'}); - $message .= ' and expected status code '; - $message .= quote($test->{'expected-status-code'}); + $message .= 'URL '; + $message .= quote($test->{'data'}); + $message .= ' and expected status code '; + $message .= quote($test->{'expected-status-code'}); - } elsif ($test->{'type'} == TRUSTED_CGI_REQUEST) { + } elsif ($test->{'type'} == TRUSTED_CGI_REQUEST) { - $message .= ' CGI URL '; - $message .= quote($test->{'data'}); - $message .= ' and expected status code '; - $message .= quote($test->{'expected-status-code'}); + $message .= 'CGI URL '; + $message .= quote($test->{'data'}); + $message .= ' and expected status code '; + $message .= quote($test->{'expected-status-code'}); - } elsif ($test->{'type'} == METHOD_TEST) { + } elsif ($test->{'type'} == METHOD_TEST) { - $message .= ' HTTP method '; - $message .= quote($test->{'data'}); - $message .= ' and expected status code '; - $message .= quote($test->{'expected-status-code'}); + $message .= 'HTTP method '; + $message .= quote($test->{'data'}); + $message .= ' and expected status code '; + $message .= quote($test->{'expected-status-code'}); - } elsif ($test->{'type'} == BLOCK_TEST) { + } elsif ($test->{'type'} == BLOCK_TEST) { - $message .= ' Supposedly-blocked URL: '; - $message .= quote($test->{'data'}); + $message .= 'Supposedly-blocked URL: '; + $message .= quote($test->{'data'}); - } elsif ($test->{'type'} == STICKY_ACTIONS_TEST) { + } elsif ($test->{'type'} == STICKY_ACTIONS_TEST) { - $message .= ' Sticky Actions: '; - $message .= quote($test->{'sticky-actions'}); - $message .= ' and URL: '; - $message .= quote($test->{'data'}); + $message .= 'Sticky Actions: '; + $message .= quote($test->{'sticky-actions'}); + $message .= ' and URL: '; + $message .= quote($test->{'data'}); - } elsif ($test->{'type'} == REDIRECT_TEST) { + } elsif ($test->{'type'} == REDIRECT_TEST) { - $message .= ' Redirected URL: '; - $message .= quote($test->{'data'}); - $message .= ' and redirect destination: '; - $message .= quote($test->{'redirect destination'}); + $message .= 'Redirected URL: '; + $message .= quote($test->{'data'}); + $message .= ' and redirect destination: '; + $message .= quote($test->{'redirect destination'}); - } else { + } else { - die "Incomplete support for test type " . $test->{'type'} . " detected."; - } + die "Incomplete support for test type " . $test->{'type'} . " detected."; } log_message($message) if (!$result or cli_option_is_set('verbose')); @@ -1598,7 +1637,7 @@ sub quote ($) { } sub print_version () { - printf PRT_VERSION . "\n" . 'Copyright (C) 2007-2011 Fabian Keil ' . "\n"; + printf PRT_VERSION . "\n"; } sub list_test_types () { @@ -1871,6 +1910,36 @@ To verify that requests for a URL get redirected, use: # Redirected URL = http://www.example.com/redirect-me # Redirect Destination = http://www.example.org/redirected +To skip a test, add the following line: + +# Ignore = Yes + +The difference between a skipped test and a removed one is that removing +a test affects the numbers of the following tests, while a skipped test +is still loaded and thus keeps the test numbers unchanged. + +Sometimes user modifications intentionally conflict with tests in the +default configuration and thus cause test failures. Adding the Ignore +directive to the failing tests works but is inconvenient as the directive +is likely to get lost with the next update. + +Overwrite conditions are an alternative and can be added in any action +file as long as the come after the test that is expected to fail. +They causes all previous tests a matching the condition to be skipped. + +It is recommended to put the overwrite condition below the custom Privoxy +section that causes the expected test failure and before the custom test +that verifies that tests the now expected behaviour. Example: + +# The following section is expected to overwrite a section in +# default.action, whose effect is tested. Thus also disable the +# test that is now expected to fail and add a new one. +# +{+block{Facebook makes Firefox even more unstable. Do not want.}} +# Overwrite condition = http://apps.facebook.com/onthefarm/track.php?creative=&cat=friendvisit&subcat=weeds&key=a789a971dc687bee4c20c044834fabdd&next=index.php%3Fref%3Dnotif%26visitId%3D898835505 +# Blocked URL = http://apps.facebook.com/ +.facebook./ + =head1 TEST LEVELS All tests have test levels to let the user @@ -1888,6 +1957,13 @@ max-level value as failed tests will result in outgoing connections. Use the B<--max-level> option to run them as well. +The "Default level offset" directive can be used to change +the default level by a given value. This directive affects +all tests located after it until the end of the file or a another +"Default level offset" directive is reached. The purpose of this +directive is to make it more convenient to skip similar tests in +a given file without having to remove or disable the tests completely. + =head1 OPTIONS B<--debug bitmask> Add the bitmask provided as integer