From de9964b3f42690cdaf4c6c73d9deedde89ceee10 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 11 Jun 2020 11:20:14 +0200 Subject: [PATCH] Add support for Brotli decompression Using Google's brotli library: https://github.com/google/brotli Sponsored by: Robert Klemme --- cgisimple.c | 8 ++++ configure.in | 23 +++++++++ filters.c | 13 +++-- parsers.c | 107 +++++++++++++++++++++++++++++++++++++++++- project.h | 3 +- templates/show-status | 5 ++ 6 files changed, 154 insertions(+), 5 deletions(-) diff --git a/cgisimple.c b/cgisimple.c index f962fdfa..3be01e39 100644 --- a/cgisimple.c +++ b/cgisimple.c @@ -1759,6 +1759,14 @@ static jb_err show_defines(struct map *exports) 1, #else 0, +#endif + }, + { + "FEATURE_BROTLI", +#ifdef FEATURE_BROTLI + 1, +#else + 0, #endif }, { diff --git a/configure.in b/configure.in index 929f9f08..687d4756 100644 --- a/configure.in +++ b/configure.in @@ -1162,6 +1162,29 @@ fi AC_SUBST(FEATURE_HTTPS_INSPECTION_ONLY) +dnl ======================================================== +dnl Check for Brotli which can be used for decompression +dnl ======================================================== +WITH_BROTLI=no +AC_ARG_WITH(brotli, +AC_HELP_STRING([--with-brotli], [Enable Brotli detection]) +AC_HELP_STRING([--without-brotli], [Disable Brotli detection]), + WITH_BROTLI=$withval) + +if test X"$WITH_BROTLI" != Xno; then + + LIBS="$LIBS -lbrotlidec" + + AC_CHECK_LIB(brotlidec, BrotliDecoderDecompress) + + AC_CHECK_HEADERS(brotli/decode.h, + FEATURE_BROTLI=1 + AC_DEFINE(FEATURE_BROTLI, 1, [If Brotli is used for decompression]) + AC_SUBST(FEATURE_BROTLI, [1]) + ) +fi + + dnl ================================================================= dnl Final cleanup and output dnl ================================================================= diff --git a/filters.c b/filters.c index c891f4f9..5c6a18a0 100644 --- a/filters.c +++ b/filters.c @@ -2166,7 +2166,11 @@ static jb_err prepare_for_filtering(struct client_state *csp) * If the body has a supported transfer-encoding, * decompress it, adjusting size and iob->eod. */ - if (csp->content_type & (CT_GZIP|CT_DEFLATE)) + if ((csp->content_type & (CT_GZIP|CT_DEFLATE)) +#ifdef FEATURE_BROTLI + || (csp->content_type & CT_BROTLI) +#endif + ) { if (0 == csp->iob->eod - csp->iob->cur) { @@ -2184,11 +2188,14 @@ static jb_err prepare_for_filtering(struct client_state *csp) else { /* - * Unset CT_GZIP and CT_DEFLATE to remember not - * to modify the Content-Encoding header later. + * Unset content types to remember not to + * modify the Content-Encoding header later. */ csp->content_type &= ~CT_GZIP; csp->content_type &= ~CT_DEFLATE; +#ifdef FEATURE_BROTLI + csp->content_type &= ~CT_BROTLI; +#endif } } #endif diff --git a/parsers.c b/parsers.c index 15abc409..5f8c6861 100644 --- a/parsers.c +++ b/parsers.c @@ -64,6 +64,9 @@ #define GZIP_FLAG_COMMENT 0x10 #define GZIP_FLAG_RESERVED_BITS 0xe0 #endif +#ifdef FEATURE_BROTLI +#include +#endif #if !defined(_WIN32) && !defined(__OS2__) #include @@ -390,6 +393,87 @@ void clear_iob(struct iob *iob) #ifdef FEATURE_ZLIB +#ifdef FEATURE_BROTLI +/********************************************************************* + * + * Function : decompress_iob_with_brotli + * + * Description : Decompress buffered page using Brotli. + * + * Parameters : + * 1 : csp = Current client state (buffers, headers, etc...) + * + * Returns : JB_ERR_OK on success, + * JB_ERR_MEMORY if out-of-memory limit reached, and + * JB_ERR_COMPRESS if error decompressing buffer. + * + *********************************************************************/ +static jb_err decompress_iob_with_brotli(struct client_state *csp) +{ + BrotliDecoderResult result; + char *decoded_buffer; + size_t decoded_size; + size_t decoded_buffer_size; + size_t encoded_size; + enum { MAX_COMPRESSION_FACTOR = 15 }; + + encoded_size = (size_t)(csp->iob->eod - csp->iob->cur); + /* + * The BrotliDecoderDecompress() api is a bit unfortunate + * and requires the caller to reserve enough memory for + * the decompressed content. Hopefully reserving + * MAX_COMPRESSION_FACTOR times the original size is + * sufficient. If not, BrotliDecoderDecompress() will fail. + */ + decoded_buffer_size = encoded_size * MAX_COMPRESSION_FACTOR; + + if (decoded_buffer_size > csp->config->buffer_limit) + { + log_error(LOG_LEVEL_ERROR, + "Buffer limit reached before decompressing iob with Brotli"); + return JB_ERR_MEMORY; + } + + decoded_buffer = malloc(decoded_buffer_size); + if (decoded_buffer == NULL) + { + log_error(LOG_LEVEL_ERROR, + "Failed to allocate %d bytes for Brotli decompression", + decoded_buffer_size); + return JB_ERR_MEMORY; + } + + decoded_size = decoded_buffer_size; + result = BrotliDecoderDecompress(encoded_size, + (const uint8_t *)csp->iob->cur, &decoded_size, + (uint8_t *)decoded_buffer); + if (result == BROTLI_DECODER_RESULT_SUCCESS) + { + /* + * Update the iob, since the decompression was successful. + */ + freez(csp->iob->buf); + csp->iob->buf = decoded_buffer; + csp->iob->cur = csp->iob->buf; + csp->iob->eod = csp->iob->cur + decoded_size; + csp->iob->size = decoded_buffer_size; + + log_error(LOG_LEVEL_RE_FILTER, + "Decompression successful. Old size: %d, new size: %d.", + encoded_size, decoded_size); + + return JB_ERR_OK; + } + else + { + log_error(LOG_LEVEL_ERROR, "Failed to decompress buffer with Brotli"); + freez(decoded_buffer); + + return JB_ERR_COMPRESS; + } +} +#endif + /********************************************************************* * * Function : decompress_iob @@ -445,6 +529,13 @@ jb_err decompress_iob(struct client_state *csp) return JB_ERR_COMPRESS; } +#ifdef FEATURE_BROTLI + if (csp->content_type & CT_BROTLI) + { + return decompress_iob_with_brotli(csp); + } +#endif + if (csp->content_type & CT_GZIP) { /* @@ -2409,6 +2500,15 @@ static jb_err server_content_encoding(struct client_state *csp, char **header) /* Mark for zlib decompression */ csp->content_type |= CT_DEFLATE; } + else if (strstr(*header, "br")) + { +#ifdef FEATURE_BROTLI + /* Mark for Brotli decompression */ + csp->content_type |= CT_BROTLI; +#else + csp->content_type |= CT_TABOO; +#endif + } else if (strstr(*header, "compress")) { /* @@ -2475,7 +2575,12 @@ static jb_err server_content_encoding(struct client_state *csp, char **header) static jb_err server_adjust_content_encoding(struct client_state *csp, char **header) { if ((csp->flags & CSP_FLAG_MODIFIED) - && (csp->content_type & (CT_GZIP | CT_DEFLATE))) + && ((csp->content_type & (CT_GZIP | CT_DEFLATE)) +#ifdef FEATURE_BROTLI + || (csp->content_type & CT_BROTLI) +#endif + ) + ) { /* * We successfully decompressed the content, diff --git a/project.h b/project.h index 943e66da..e241360a 100644 --- a/project.h +++ b/project.h @@ -491,13 +491,14 @@ struct iob */ #define CT_GZIP 0x0010U /**< gzip-compressed data. */ #define CT_DEFLATE 0x0020U /**< zlib-compressed data. */ +#define CT_BROTLI 0x0040U /**< Brotli-compressed data. */ /** * Flag to signal that the server declared the content type, * so we can differentiate between unknown and undeclared * content types. */ -#define CT_DECLARED 0x0040U +#define CT_DECLARED 0x0080U /** * The mask which includes all actions. diff --git a/templates/show-status b/templates/show-status index 425abbd5..67d94086 100644 --- a/templates/show-status +++ b/templates/show-status @@ -237,6 +237,11 @@ @if-FEATURE_ACL-then@ Yes @else-not-FEATURE_ACL@ No @endif-FEATURE_ACL@ Allows the use of an ACL to control access to Privoxy by IP address. + + FEATURE_BROTLI + @if-FEATURE_BROTLI-then@ Yes @else-not-FEATURE_BROTLI@ No @endif-FEATURE_BROTLI@ + Allows to decompress content with Brotli before filtering it. Requires external brotli library. + FEATURE_CGI_EDIT_ACTIONS @if-FEATURE_CGI_EDIT_ACTIONS-then@ Yes @else-not-FEATURE_CGI_EDIT_ACTIONS@ No @endif-FEATURE_CGI_EDIT_ACTIONS@ -- 2.39.2