Merge branch 'master' of ssh://git.privoxy.org:23/git/privoxy

diff --git a/GNUmakefile.in b/GNUmakefile.in
index 92ef0a1..40b5706 100644 (file)
--- a/GNUmakefile.in
+++ b/GNUmakefile.in
@@ -280,7 +280,6 @@ SPECIAL_CFLAGS = @SPECIAL_CFLAGS@
 OTHER_CFLAGS =
 
 CFLAGS = @CFLAGS@ @CPPFLAGS@ $(OTHER_CFLAGS) $(SPECIAL_CFLAGS) -Wall \
-         -Imbedtls/include \
          @STATIC_PCRE_ONLY@ -Ipcre
 
 LDFLAGS = @LDFLAGS@ $(DEBUG_CFLAGS) $(SPECIAL_CFLAGS)

diff --git a/TODO b/TODO
index 424a6e5..11b58ca 100644 (file)
--- a/TODO
+++ b/TODO
@@ -443,8 +443,6 @@ https://www.privoxy.org/faq/general.html#DONATE
      memory footprint a bit which may be noticeable in case of multi-user setups
      with hundreds of idle connections.
 
-159) Support Brotli compression.
-
 160) Add keep-alive support with +https-inspection.
 
 ##########################################################################

diff --git a/cgisimple.c b/cgisimple.c
index f962fdf..3be01e3 100644 (file)
--- a/cgisimple.c
+++ b/cgisimple.c
@@ -1759,6 +1759,14 @@ static jb_err show_defines(struct map *exports)
          1,
 #else
          0,
+#endif
+      },
+      {
+         "FEATURE_BROTLI",
+#ifdef FEATURE_BROTLI
+         1,
+#else
+         0,
 #endif
       },
       {

diff --git a/configure.in b/configure.in
index 2889ac2..2ca7e24 100644 (file)
--- a/configure.in
+++ b/configure.in
@@ -1,6 +1,6 @@
 dnl Process this file with autoconf to produce a configure script.
 dnl
-dnl Written by and Copyright (C) 2001-2019 the
+dnl Written by and Copyright (C) 2001-2020 the
 dnl Privoxy team. https://www.privoxy.org/
 dnl
 dnl Based on the Internet Junkbuster originally written
@@ -1137,8 +1137,8 @@ dnl ========================================================
 FEATURE_HTTPS_INSPECTION_ONLY=#
 OPT_MBEDTLS=no
 AC_ARG_WITH(mbedtls,dnl
-AC_HELP_STRING([--with-mbedtls],[enable mbedTLS detection for https inspection.])
-AC_HELP_STRING([--without-mbedtls], [disable mbedTLS detection]),
+AC_HELP_STRING([--with-mbedtls], [Enable mbedTLS detection for https inspection.])
+AC_HELP_STRING([--without-mbedtls], [Disable mbedTLS detection]),
   OPT_MBEDTLS=$withval)
 
 if test X"$OPT_MBEDTLS" != Xno; then
@@ -1154,6 +1154,7 @@ if test X"$OPT_MBEDTLS" != Xno; then
     AC_MSG_NOTICE([Detected mbedTLS. Enabling https inspection.])
 
     LIBS="-lmbedtls -lmbedx509 -lmbedcrypto $LIBS"
+    old_CFLAGS_nospecial="-Imbedtls/include $old_CFLAGS_nospecial"
 
     FEATURE_HTTPS_INSPECTION_ONLY=
   fi
@@ -1161,6 +1162,29 @@ fi
 AC_SUBST(FEATURE_HTTPS_INSPECTION_ONLY)
 
 
+dnl ========================================================
+dnl Check for Brotli which can be used for decompression
+dnl ========================================================
+WITH_BROTLI=no
+AC_ARG_WITH(brotli,
+AC_HELP_STRING([--with-brotli], [Enable Brotli detection])
+AC_HELP_STRING([--without-brotli], [Disable Brotli detection]),
+  WITH_BROTLI=$withval)
+
+if test X"$WITH_BROTLI" != Xno; then
+
+  LIBS="$LIBS -lbrotlidec"
+
+  AC_CHECK_LIB(brotlidec, BrotliDecoderDecompress)
+
+  AC_CHECK_HEADERS(brotli/decode.h,
+    FEATURE_BROTLI=1
+    AC_DEFINE(FEATURE_BROTLI, 1, [If Brotli is used for decompression])
+    AC_SUBST(FEATURE_BROTLI, [1])
+  )
+fi
+
+
 dnl =================================================================
 dnl Final cleanup and output
 dnl =================================================================

diff --git a/default.action.master b/default.action.master
index 99eb553..dc7c681 100644 (file)
--- a/default.action.master
+++ b/default.action.master
@@ -555,7 +555,6 @@ allow-ads   = -block -filter{banners-by-size} -filter{banners-by-link}
 #
 { \
 +change-x-forwarded-for{block} \
-+client-header-filter{no-brotli-accepted} \
 +client-header-tagger{css-requests} \
 +client-header-tagger{image-requests} \
 +client-header-tagger{range-requests} \
@@ -570,7 +569,6 @@ standard.Cautious
 #
 { \
 +change-x-forwarded-for{block} \
-+client-header-filter{no-brotli-accepted} \
 +client-header-tagger{css-requests} \
 +client-header-tagger{image-requests} \
 +client-header-tagger{range-requests} \
@@ -601,7 +599,6 @@ standard.Medium
 #
 { \
 +change-x-forwarded-for{block} \
-+client-header-filter{no-brotli-accepted} \
 +client-header-tagger{css-requests} \
 +client-header-tagger{image-requests} \
 +client-header-tagger{range-requests} \
@@ -963,6 +960,8 @@ go.idmnet.bbelements.com/please/showit/
 gm-link.com/
 # Blocked URL = https://collector.githubapp.com/github/page_view?dimensions[page]=https%3A%2F%2Fgithub.com%2Fopenbsd%2Fsrc%2Fblob%2F5ecdd0566b441ae0b99e73f410875e05dc0fa5b7%2Flib%2Flibc%2Finclude%2Fthread_private.h&dimensions[title]=src%2Fthread_private.h%20at%205ecdd0566b441ae0b99e73f410875e05dc0fa5b7%20%C2%B7%20openbsd%2Fsrc%20%C2%B7%20GitHub&dimensions[referrer]=https%3A%2F%2Fgithub.com%2Fopenbsd%2Fsrc%2Fsearch%3Fq%3D_MUTEX_LOCK%26unscoped_q%3D_MUTEX_LOCK&dimensions[user_agent]=Mozilla%2F5.0%20(X11%3B%20ElectroBSD%20amd64%3B%20rv%3A68.0)%20Gecko%2F20100101%20Firefox%2F68.0&dimensions[screen_resolution]=1366x768&dimensions[pixel_ratio]=1&dimensions[browser_resolution]=1362x636&dimensions[tz_seconds]=7200&dimensions[timestamp]=1591515169218&dimensions[referrer]=&dimensions[request_id]=22CA%3AE6F6%3A271ED2A%3A3839C5D%3A5EDB8295&dimensions[visitor_id]=5184934927058829974&dimensions[region_edge]=ams&dimensions[region_render]=ams&dimensions[user_id]=929183&dimensions[user_login]=openbsd&dimensions[repository_id]=66966208&dimensions[repository_nwo]=openbsd%2Fsrc&dimensions[repository_public]=true&dimensions[repository_is_fork]=false&dimensions[repository_network_root_id]=66966208&dimensions[repository_network_root_nwo]=openbsd%2Fsrc&dimensions[repository_explore_github_marketplace_ci_cta_shown]=false&&measures[performance_timing]=1-172-172-13996-12735-12426-12394-3858-172-172-172--13996-0---172-3853-3412-0--&&&dimensions[cid]=268614077.1591515169
 collector.githubapp.com/
+# Blocked URL = https://consentmanager.mgr.consensu.org/delivery/pixel.php?id=11319&did=0&cfdid=0&t=pv&h=https%3A%2F%2Fsourceforge.net%2Fp%2Fijbswa%2Ffeature-requests%2F535%2F&o=1591880198219&l=EN&lv=0&d=0&ct=14&e=&e2=&e3=&i=&sv=0&dv=0
+.consensu.org/delivery/pixel\.php
 
 
 #############################################################################
@@ -2166,6 +2165,10 @@ oss-fuzz.com/
 sourcepoint.vice.com/
 # URL = https://collector.githubapp.com/github/page_view?dimensions[page]=https%3A%2F%2Fgithub.com%2Fopenbsd%2Fsrc%2Fblob%2F5ecdd0566b441ae0b99e73f410875e05dc0fa5b7%2Flib%2Flibc%2Finclude%2Fthread_private.h&dimensions[title]=src%2Fthread_private.h%20at%205ecdd0566b441ae0b99e73f410875e05dc0fa5b7%20%C2%B7%20openbsd%2Fsrc%20%C2%B7%20GitHub&dimensions[referrer]=https%3A%2F%2Fgithub.com%2Fopenbsd%2Fsrc%2Fsearch%3Fq%3D_MUTEX_LOCK%26unscoped_q%3D_MUTEX_LOCK&dimensions[user_agent]=Mozilla%2F5.0%20(X11%3B%20ElectroBSD%20amd64%3B%20rv%3A68.0)%20Gecko%2F20100101%20Firefox%2F68.0&dimensions[screen_resolution]=1366x768&dimensions[pixel_ratio]=1&dimensions[browser_resolution]=1362x636&dimensions[tz_seconds]=7200&dimensions[timestamp]=1591515169218&dimensions[referrer]=&dimensions[request_id]=22CA%3AE6F6%3A271ED2A%3A3839C5D%3A5EDB8295&dimensions[visitor_id]=5184934927058829974&dimensions[region_edge]=ams&dimensions[region_render]=ams&dimensions[user_id]=929183&dimensions[user_login]=openbsd&dimensions[repository_id]=66966208&dimensions[repository_nwo]=openbsd%2Fsrc&dimensions[repository_public]=true&dimensions[repository_is_fork]=false&dimensions[repository_network_root_id]=66966208&dimensions[repository_network_root_nwo]=openbsd%2Fsrc&dimensions[repository_explore_github_marketplace_ci_cta_shown]=false&&measures[performance_timing]=1-172-172-13996-12735-12426-12394-3858-172-172-172--13996-0---172-3853-3412-0--&&&dimensions[cid]=268614077.1591515169
 collector.githubapp.com/
+# URL = https://consentmanager.mgr.consensu.org/delivery/pixel.php?id=11319&did=0&cfdid=0&t=pv&h=https%3A%2F%2Fsourceforge.net%2Fp%2Fijbswa%2Ffeature-requests%2F535%2F&o=1591880198219&l=EN&lv=0&d=0&ct=14&e=&e2=&e3=&i=&sv=0&dv=0
+.consensu.org/delivery/pixel\.php
+# URL = https://de.wikipedia.org/beacon/event?%7B%22event%22%3A%7B%22source_page_id%22%3A1501869%2C%22source_namespace%22%3A0%2C%22source_title%22%3A%22Primat%22%2C%22source_url%22%3A%22https%3A%2F%2Fde.wikipedia.org%2Fwiki%2FPrimat%22%2C%22page_title%22%3A%22Primaten%22%2C%22page_id%22%3A4022%2C%22page_namespace%22%3A0%7D%2C%22schema%22%3A%22VirtualPageView%22%2C%22webHost%22%3A%22de.wikipedia.org%22%2C%22wiki%22%3A%22dewiki%22%2C%22revision%22%3A17780078%7D;
+.wikipedia.org/
 
 {+redirect{s@.*url=@http://@} -block}
 # Sticky Actions = +redirect -block

diff --git a/filters.c b/filters.c
index c891f4f..b490353 100644 (file)
--- a/filters.c
+++ b/filters.c
@@ -4,7 +4,7 @@
  *
  * Purpose     :  Declares functions to parse/crunch headers and pages.
  *
- * Copyright   :  Written by and Copyright (C) 2001-2016 the
+ * Copyright   :  Written by and Copyright (C) 2001-2020 the
  *                Privoxy team. https://www.privoxy.org/
  *
  *                Based on the Internet Junkbuster originally written
@@ -2166,7 +2166,11 @@ static jb_err prepare_for_filtering(struct client_state *csp)
     * If the body has a supported transfer-encoding,
     * decompress it, adjusting size and iob->eod.
     */
-   if (csp->content_type & (CT_GZIP|CT_DEFLATE))
+   if ((csp->content_type & (CT_GZIP|CT_DEFLATE))
+#ifdef FEATURE_BROTLI
+      || (csp->content_type & CT_BROTLI)
+#endif
+       )
    {
       if (0 == csp->iob->eod - csp->iob->cur)
       {
@@ -2184,11 +2188,14 @@ static jb_err prepare_for_filtering(struct client_state *csp)
       else
       {
          /*
-          * Unset CT_GZIP and CT_DEFLATE to remember not
-          * to modify the Content-Encoding header later.
+          * Unset content types to remember not to
+          * modify the Content-Encoding header later.
           */
          csp->content_type &= ~CT_GZIP;
          csp->content_type &= ~CT_DEFLATE;
+#ifdef FEATURE_BROTLI
+         csp->content_type &= ~CT_BROTLI;
+#endif
       }
    }
 #endif

diff --git a/jcc.c b/jcc.c
index 6d91bd0..69c0180 100644 (file)
--- a/jcc.c
+++ b/jcc.c
@@ -2220,7 +2220,8 @@ static jb_err receive_encrypted_request(struct client_state *csp)
    do
    {
       log_error(LOG_LEVEL_HEADER, "Reading encrypted headers");
-      if (!data_is_available(csp->cfd, (int)csp->config->keep_alive_timeout))
+      if (!is_ssl_pending(&(csp->mbedtls_client_attr.ssl)) &&
+          !data_is_available(csp->cfd, csp->config->socket_timeout))
       {
          log_error(LOG_LEVEL_CONNECT,
             "Socket %d timed out while waiting for client headers", csp->cfd);

diff --git a/match-all.action b/match-all.action
index d7bbc6b..37520d3 100644 (file)
--- a/match-all.action
+++ b/match-all.action
@@ -7,7 +7,6 @@
 #############################################################################
 { \
 +change-x-forwarded-for{block} \
-+client-header-filter{no-brotli-accepted} \
 +client-header-tagger{css-requests} \
 +client-header-tagger{image-requests} \
 +client-header-tagger{range-requests} \

diff --git a/parsers.c b/parsers.c
index 15abc40..5f8c686 100644 (file)
--- a/parsers.c
+++ b/parsers.c
@@ -64,6 +64,9 @@
 #define GZIP_FLAG_COMMENT       0x10
 #define GZIP_FLAG_RESERVED_BITS 0xe0
 #endif
+#ifdef FEATURE_BROTLI
+#include <brotli/decode.h>
+#endif
 
 #if !defined(_WIN32) && !defined(__OS2__)
 #include <unistd.h>
@@ -390,6 +393,87 @@ void clear_iob(struct iob *iob)
 
 
 #ifdef FEATURE_ZLIB
+#ifdef FEATURE_BROTLI
+/*********************************************************************
+ *
+ * Function    :  decompress_iob_with_brotli
+ *
+ * Description :  Decompress buffered page using Brotli.
+ *
+ * Parameters  :
+ *          1  :  csp = Current client state (buffers, headers, etc...)
+ *
+ * Returns     :  JB_ERR_OK on success,
+ *                JB_ERR_MEMORY if out-of-memory limit reached, and
+ *                JB_ERR_COMPRESS if error decompressing buffer.
+ *
+ *********************************************************************/
+static jb_err decompress_iob_with_brotli(struct client_state *csp)
+{
+   BrotliDecoderResult result;
+   char *decoded_buffer;
+   size_t decoded_size;
+   size_t decoded_buffer_size;
+   size_t encoded_size;
+   enum { MAX_COMPRESSION_FACTOR = 15 };
+
+   encoded_size = (size_t)(csp->iob->eod - csp->iob->cur);
+   /*
+    * The BrotliDecoderDecompress() api is a bit unfortunate
+    * and requires the caller to reserve enough memory for
+    * the decompressed content. Hopefully reserving
+    * MAX_COMPRESSION_FACTOR times the original size is
+    * sufficient. If not, BrotliDecoderDecompress() will fail.
+    */
+   decoded_buffer_size = encoded_size * MAX_COMPRESSION_FACTOR;
+
+   if (decoded_buffer_size > csp->config->buffer_limit)
+   {
+      log_error(LOG_LEVEL_ERROR,
+         "Buffer limit reached before decompressing iob with Brotli");
+      return JB_ERR_MEMORY;
+   }
+
+   decoded_buffer = malloc(decoded_buffer_size);
+   if (decoded_buffer == NULL)
+   {
+      log_error(LOG_LEVEL_ERROR,
+         "Failed to allocate %d bytes for Brotli decompression",
+         decoded_buffer_size);
+      return JB_ERR_MEMORY;
+   }
+
+   decoded_size = decoded_buffer_size;
+   result = BrotliDecoderDecompress(encoded_size,
+      (const uint8_t *)csp->iob->cur, &decoded_size,
+      (uint8_t *)decoded_buffer);
+   if (result == BROTLI_DECODER_RESULT_SUCCESS)
+   {
+      /*
+       * Update the iob, since the decompression was successful.
+       */
+      freez(csp->iob->buf);
+      csp->iob->buf  = decoded_buffer;
+      csp->iob->cur  = csp->iob->buf;
+      csp->iob->eod  = csp->iob->cur + decoded_size;
+      csp->iob->size = decoded_buffer_size;
+
+      log_error(LOG_LEVEL_RE_FILTER,
+         "Decompression successful. Old size: %d, new size: %d.",
+         encoded_size, decoded_size);
+
+      return JB_ERR_OK;
+   }
+   else
+   {
+      log_error(LOG_LEVEL_ERROR, "Failed to decompress buffer with Brotli");
+      freez(decoded_buffer);
+
+      return JB_ERR_COMPRESS;
+   }
+}
+#endif
+
 /*********************************************************************
  *
  * Function    :  decompress_iob
@@ -445,6 +529,13 @@ jb_err decompress_iob(struct client_state *csp)
       return JB_ERR_COMPRESS;
    }
 
+#ifdef FEATURE_BROTLI
+   if (csp->content_type & CT_BROTLI)
+   {
+      return decompress_iob_with_brotli(csp);
+   }
+#endif
+
    if (csp->content_type & CT_GZIP)
    {
       /*
@@ -2409,6 +2500,15 @@ static jb_err server_content_encoding(struct client_state *csp, char **header)
       /* Mark for zlib decompression */
       csp->content_type |= CT_DEFLATE;
    }
+   else if (strstr(*header, "br"))
+   {
+#ifdef FEATURE_BROTLI
+      /* Mark for Brotli decompression */
+      csp->content_type |= CT_BROTLI;
+#else
+      csp->content_type |= CT_TABOO;
+#endif
+   }
    else if (strstr(*header, "compress"))
    {
       /*
@@ -2475,7 +2575,12 @@ static jb_err server_content_encoding(struct client_state *csp, char **header)
 static jb_err server_adjust_content_encoding(struct client_state *csp, char **header)
 {
    if ((csp->flags & CSP_FLAG_MODIFIED)
-    && (csp->content_type & (CT_GZIP | CT_DEFLATE)))
+      && ((csp->content_type & (CT_GZIP | CT_DEFLATE))
+#ifdef FEATURE_BROTLI
+         || (csp->content_type & CT_BROTLI)
+#endif
+         )
+      )
    {
       /*
        * We successfully decompressed the content,

diff --git a/project.h b/project.h
index 943e66d..e241360 100644 (file)
--- a/project.h
+++ b/project.h
@@ -491,13 +491,14 @@ struct iob
  */
 #define CT_GZIP    0x0010U /**< gzip-compressed data. */
 #define CT_DEFLATE 0x0020U /**< zlib-compressed data. */
+#define CT_BROTLI  0x0040U /**< Brotli-compressed data. */
 
 /**
  * Flag to signal that the server declared the content type,
  * so we can differentiate between unknown and undeclared
  * content types.
  */
-#define CT_DECLARED 0x0040U
+#define CT_DECLARED 0x0080U
 
 /**
  * The mask which includes all actions.

diff --git a/ssl.c b/ssl.c
index 313174f..860a243 100644 (file)
--- a/ssl.c
+++ b/ssl.c
@@ -2064,7 +2064,12 @@ extern void ssl_send_certificate_error(struct client_state *csp)
       "HTTP/1.1 200 OK\r\n"
       "Content-Type: text/html\r\n"
       "Connection: close\r\n\r\n"
-      "<html><body><h1>Server certificate verification failed</h1><p>Reason: ";
+      "<!DOCTYPE html>\n"
+      "<html><head><title>Server certificate verification failed</title></head>\n"
+      "<body><h1>Server certificate verification failed</h1>\n"
+      "<p><a href=\"https://" CGI_SITE_2_HOST "/\">Privoxy</a> was unable "
+      "to securely connnect to the destination server.</p>"
+      "<p>Reason: ";
    const char message_end[] = "</body></html>\r\n\r\n";
    char reason[INVALID_CERT_INFO_BUF_SIZE];
    memset(reason, 0, sizeof(reason));

diff --git a/templates/show-status b/templates/show-status
index 425abbd..67d9408 100644 (file)
--- a/templates/show-status
+++ b/templates/show-status
@@ -237,6 +237,11 @@
               <td>@if-FEATURE_ACL-then@ Yes @else-not-FEATURE_ACL@ No @endif-FEATURE_ACL@</td>
               <td>Allows the use of an ACL to control access to Privoxy by IP address.</td>
             </tr>
+            <tr>
+              <td><code>FEATURE_BROTLI</code></td>
+              <td>@if-FEATURE_BROTLI-then@ Yes @else-not-FEATURE_BROTLI@ No @endif-FEATURE_BROTLI@</td>
+              <td>Allows to decompress content with Brotli before filtering it. Requires external brotli library.</td>
+            </tr>
             <tr>
               <td><code>FEATURE_CGI_EDIT_ACTIONS</code></td>
               <td>@if-FEATURE_CGI_EDIT_ACTIONS-then@ Yes @else-not-FEATURE_CGI_EDIT_ACTIONS@ No @endif-FEATURE_CGI_EDIT_ACTIONS@</td>