Add support for Brotli decompression

Using Google's brotli library:
https://github.com/google/brotli

Sponsored by: Robert Klemme

diff --git a/cgisimple.c b/cgisimple.c
index f962fdf..3be01e3 100644 (file)
--- a/cgisimple.c
+++ b/cgisimple.c
@@ -1759,6 +1759,14 @@ static jb_err show_defines(struct map *exports)
          1,
 #else
          0,
          1,
 #else
          0,

+#endif
+      },
+      {
+         "FEATURE_BROTLI",
+#ifdef FEATURE_BROTLI
+         1,
+#else
+         0,

 #endif
       },
       {
 #endif
       },
       {

diff --git a/configure.in b/configure.in
index 929f9f0..687d475 100644 (file)
--- a/configure.in
+++ b/configure.in
@@ -1162,6 +1162,29 @@ fi
 AC_SUBST(FEATURE_HTTPS_INSPECTION_ONLY)
 
 
 AC_SUBST(FEATURE_HTTPS_INSPECTION_ONLY)
 
 

+dnl ========================================================
+dnl Check for Brotli which can be used for decompression
+dnl ========================================================
+WITH_BROTLI=no
+AC_ARG_WITH(brotli,
+AC_HELP_STRING([--with-brotli], [Enable Brotli detection])
+AC_HELP_STRING([--without-brotli], [Disable Brotli detection]),
+  WITH_BROTLI=$withval)
+
+if test X"$WITH_BROTLI" != Xno; then
+
+  LIBS="$LIBS -lbrotlidec"
+
+  AC_CHECK_LIB(brotlidec, BrotliDecoderDecompress)
+
+  AC_CHECK_HEADERS(brotli/decode.h,
+    FEATURE_BROTLI=1
+    AC_DEFINE(FEATURE_BROTLI, 1, [If Brotli is used for decompression])
+    AC_SUBST(FEATURE_BROTLI, [1])
+  )
+fi
+
+

 dnl =================================================================
 dnl Final cleanup and output
 dnl =================================================================
 dnl =================================================================
 dnl Final cleanup and output
 dnl =================================================================

diff --git a/filters.c b/filters.c
index c891f4f..5c6a18a 100644 (file)
--- a/filters.c
+++ b/filters.c
@@ -2166,7 +2166,11 @@ static jb_err prepare_for_filtering(struct client_state *csp)
     * If the body has a supported transfer-encoding,
     * decompress it, adjusting size and iob->eod.
     */
     * If the body has a supported transfer-encoding,
     * decompress it, adjusting size and iob->eod.
     */

-   if (csp->content_type & (CT_GZIP|CT_DEFLATE))
+   if ((csp->content_type & (CT_GZIP|CT_DEFLATE))
+#ifdef FEATURE_BROTLI
+      || (csp->content_type & CT_BROTLI)
+#endif
+       )

    {
       if (0 == csp->iob->eod - csp->iob->cur)
       {
    {
       if (0 == csp->iob->eod - csp->iob->cur)
       {


@@ -2184,11 +2188,14 @@ static jb_err prepare_for_filtering(struct client_state *csp)
       else
       {
          /*
       else
       {
          /*

-          * Unset CT_GZIP and CT_DEFLATE to remember not
-          * to modify the Content-Encoding header later.
+          * Unset content types to remember not to
+          * modify the Content-Encoding header later.

           */
          csp->content_type &= ~CT_GZIP;
          csp->content_type &= ~CT_DEFLATE;
           */
          csp->content_type &= ~CT_GZIP;
          csp->content_type &= ~CT_DEFLATE;

+#ifdef FEATURE_BROTLI
+         csp->content_type &= ~CT_BROTLI;
+#endif

       }
    }
 #endif
       }
    }
 #endif

diff --git a/parsers.c b/parsers.c
index 15abc40..5f8c686 100644 (file)
--- a/parsers.c
+++ b/parsers.c
@@ -64,6 +64,9 @@
 #define GZIP_FLAG_COMMENT       0x10
 #define GZIP_FLAG_RESERVED_BITS 0xe0
 #endif
 #define GZIP_FLAG_COMMENT       0x10
 #define GZIP_FLAG_RESERVED_BITS 0xe0
 #endif

+#ifdef FEATURE_BROTLI
+#include <brotli/decode.h>
+#endif

 
 #if !defined(_WIN32) && !defined(__OS2__)
 #include <unistd.h>
 
 #if !defined(_WIN32) && !defined(__OS2__)
 #include <unistd.h>


@@ -390,6 +393,87 @@ void clear_iob(struct iob *iob)
 
 
 #ifdef FEATURE_ZLIB
 
 
 #ifdef FEATURE_ZLIB

+#ifdef FEATURE_BROTLI
+/*********************************************************************
+ *
+ * Function    :  decompress_iob_with_brotli
+ *
+ * Description :  Decompress buffered page using Brotli.
+ *
+ * Parameters  :
+ *          1  :  csp = Current client state (buffers, headers, etc...)
+ *
+ * Returns     :  JB_ERR_OK on success,
+ *                JB_ERR_MEMORY if out-of-memory limit reached, and
+ *                JB_ERR_COMPRESS if error decompressing buffer.
+ *
+ *********************************************************************/
+static jb_err decompress_iob_with_brotli(struct client_state *csp)
+{
+   BrotliDecoderResult result;
+   char *decoded_buffer;
+   size_t decoded_size;
+   size_t decoded_buffer_size;
+   size_t encoded_size;
+   enum { MAX_COMPRESSION_FACTOR = 15 };
+
+   encoded_size = (size_t)(csp->iob->eod - csp->iob->cur);
+   /*
+    * The BrotliDecoderDecompress() api is a bit unfortunate
+    * and requires the caller to reserve enough memory for
+    * the decompressed content. Hopefully reserving
+    * MAX_COMPRESSION_FACTOR times the original size is
+    * sufficient. If not, BrotliDecoderDecompress() will fail.
+    */
+   decoded_buffer_size = encoded_size * MAX_COMPRESSION_FACTOR;
+
+   if (decoded_buffer_size > csp->config->buffer_limit)
+   {
+      log_error(LOG_LEVEL_ERROR,
+         "Buffer limit reached before decompressing iob with Brotli");
+      return JB_ERR_MEMORY;
+   }
+
+   decoded_buffer = malloc(decoded_buffer_size);
+   if (decoded_buffer == NULL)
+   {
+      log_error(LOG_LEVEL_ERROR,
+         "Failed to allocate %d bytes for Brotli decompression",
+         decoded_buffer_size);
+      return JB_ERR_MEMORY;
+   }
+
+   decoded_size = decoded_buffer_size;
+   result = BrotliDecoderDecompress(encoded_size,
+      (const uint8_t *)csp->iob->cur, &decoded_size,
+      (uint8_t *)decoded_buffer);
+   if (result == BROTLI_DECODER_RESULT_SUCCESS)
+   {
+      /*
+       * Update the iob, since the decompression was successful.
+       */
+      freez(csp->iob->buf);
+      csp->iob->buf  = decoded_buffer;
+      csp->iob->cur  = csp->iob->buf;
+      csp->iob->eod  = csp->iob->cur + decoded_size;
+      csp->iob->size = decoded_buffer_size;
+
+      log_error(LOG_LEVEL_RE_FILTER,
+         "Decompression successful. Old size: %d, new size: %d.",
+         encoded_size, decoded_size);
+
+      return JB_ERR_OK;
+   }
+   else
+   {
+      log_error(LOG_LEVEL_ERROR, "Failed to decompress buffer with Brotli");
+      freez(decoded_buffer);
+
+      return JB_ERR_COMPRESS;
+   }
+}
+#endif
+

 /*********************************************************************
  *
  * Function    :  decompress_iob
 /*********************************************************************
  *
  * Function    :  decompress_iob


@@ -445,6 +529,13 @@ jb_err decompress_iob(struct client_state *csp)
       return JB_ERR_COMPRESS;
    }
 
       return JB_ERR_COMPRESS;
    }
 

+#ifdef FEATURE_BROTLI
+   if (csp->content_type & CT_BROTLI)
+   {
+      return decompress_iob_with_brotli(csp);
+   }
+#endif
+

    if (csp->content_type & CT_GZIP)
    {
       /*
    if (csp->content_type & CT_GZIP)
    {
       /*


@@ -2409,6 +2500,15 @@ static jb_err server_content_encoding(struct client_state *csp, char **header)
       /* Mark for zlib decompression */
       csp->content_type |= CT_DEFLATE;
    }
       /* Mark for zlib decompression */
       csp->content_type |= CT_DEFLATE;
    }

+   else if (strstr(*header, "br"))
+   {
+#ifdef FEATURE_BROTLI
+      /* Mark for Brotli decompression */
+      csp->content_type |= CT_BROTLI;
+#else
+      csp->content_type |= CT_TABOO;
+#endif
+   }

    else if (strstr(*header, "compress"))
    {
       /*
    else if (strstr(*header, "compress"))
    {
       /*


@@ -2475,7 +2575,12 @@ static jb_err server_content_encoding(struct client_state *csp, char **header)
 static jb_err server_adjust_content_encoding(struct client_state *csp, char **header)
 {
    if ((csp->flags & CSP_FLAG_MODIFIED)
 static jb_err server_adjust_content_encoding(struct client_state *csp, char **header)
 {
    if ((csp->flags & CSP_FLAG_MODIFIED)

-    && (csp->content_type & (CT_GZIP | CT_DEFLATE)))
+      && ((csp->content_type & (CT_GZIP | CT_DEFLATE))
+#ifdef FEATURE_BROTLI
+         || (csp->content_type & CT_BROTLI)
+#endif
+         )
+      )

    {
       /*
        * We successfully decompressed the content,
    {
       /*
        * We successfully decompressed the content,

diff --git a/project.h b/project.h
index 943e66d..e241360 100644 (file)
--- a/project.h
+++ b/project.h
@@ -491,13 +491,14 @@ struct iob
  */
 #define CT_GZIP    0x0010U /**< gzip-compressed data. */
 #define CT_DEFLATE 0x0020U /**< zlib-compressed data. */
  */
 #define CT_GZIP    0x0010U /**< gzip-compressed data. */
 #define CT_DEFLATE 0x0020U /**< zlib-compressed data. */

+#define CT_BROTLI  0x0040U /**< Brotli-compressed data. */

 
 /**
  * Flag to signal that the server declared the content type,
  * so we can differentiate between unknown and undeclared
  * content types.
  */
 
 /**
  * Flag to signal that the server declared the content type,
  * so we can differentiate between unknown and undeclared
  * content types.
  */

-#define CT_DECLARED 0x0040U
+#define CT_DECLARED 0x0080U

 
 /**
  * The mask which includes all actions.
 
 /**
  * The mask which includes all actions.

diff --git a/templates/show-status b/templates/show-status
index 425abbd..67d9408 100644 (file)
--- a/templates/show-status
+++ b/templates/show-status
@@ -237,6 +237,11 @@
               <td>@if-FEATURE_ACL-then@ Yes @else-not-FEATURE_ACL@ No @endif-FEATURE_ACL@</td>
               <td>Allows the use of an ACL to control access to Privoxy by IP address.</td>
             </tr>
               <td>@if-FEATURE_ACL-then@ Yes @else-not-FEATURE_ACL@ No @endif-FEATURE_ACL@</td>
               <td>Allows the use of an ACL to control access to Privoxy by IP address.</td>
             </tr>

+            <tr>
+              <td><code>FEATURE_BROTLI</code></td>
+              <td>@if-FEATURE_BROTLI-then@ Yes @else-not-FEATURE_BROTLI@ No @endif-FEATURE_BROTLI@</td>
+              <td>Allows to decompress content with Brotli before filtering it. Requires external brotli library.</td>
+            </tr>

             <tr>
               <td><code>FEATURE_CGI_EDIT_ACTIONS</code></td>
               <td>@if-FEATURE_CGI_EDIT_ACTIONS-then@ Yes @else-not-FEATURE_CGI_EDIT_ACTIONS@ No @endif-FEATURE_CGI_EDIT_ACTIONS@</td>
             <tr>
               <td><code>FEATURE_CGI_EDIT_ACTIONS</code></td>
               <td>@if-FEATURE_CGI_EDIT_ACTIONS-then@ Yes @else-not-FEATURE_CGI_EDIT_ACTIONS@ No @endif-FEATURE_CGI_EDIT_ACTIONS@</td>