From: Fabian Keil Date: Sun, 26 Jan 2025 13:12:23 +0000 (+0100) Subject: Add Zstandard-decompression support X-Git-Url: http://www.privoxy.org/gitweb/%22https:/@default-cgi@/user-manual/faq/static/@default-cgi@toggle?a=commitdiff_plain;h=15c91bb53616e74ddc33bcf3386611dbea31feeb;p=privoxy.git Add Zstandard-decompression support Using the reference library zstd: https://facebook.github.io/zstd/ --- diff --git a/cgisimple.c b/cgisimple.c index 961510f2..58e5ae6b 100644 --- a/cgisimple.c +++ b/cgisimple.c @@ -2202,7 +2202,15 @@ static jb_err show_defines(struct map *exports) #else 0, #endif - } + }, + { + "FEATURE_ZSTD", +#ifdef FEATURE_ZSTD + 1, +#else + 0, +#endif + }, }; for (i = 0; i < SZ(features); i++) diff --git a/configure.in b/configure.in index 51d1b20a..094ccaa6 100644 --- a/configure.in +++ b/configure.in @@ -1317,6 +1317,28 @@ if test X"$WITH_BROTLI" != Xno; then fi +dnl ======================================================== +dnl Check for zstd which can be used for decompression +dnl ======================================================== +WITH_ZSTD=no +AC_ARG_WITH(zstd, +AC_HELP_STRING([--with-zstd], [Enable zstd detection]) +AC_HELP_STRING([--without-zstd], [Disable zstd detection]), + WITH_ZSTD=$withval) + +if test X"$WITH_ZSTD" != Xno; then + + LIBS="$LIBS -lzstd" + AC_CHECK_LIB(zstd, ZSTD_decompressStream) + + AC_CHECK_HEADERS(zstd.h, + FEATURE_ZSTD=1 + AC_DEFINE(FEATURE_ZSTD, 1, [If zstd is used for decompression]) + AC_SUBST(FEATURE_ZSTD, [1]) + ) +fi + + dnl ================================================================= dnl Final cleanup and output dnl ================================================================= diff --git a/filters.c b/filters.c index 7f4a3e46..8daa8c54 100644 --- a/filters.c +++ b/filters.c @@ -2485,6 +2485,9 @@ static jb_err prepare_for_filtering(struct client_state *csp) if ((csp->content_type & (CT_GZIP|CT_DEFLATE)) #ifdef FEATURE_BROTLI || (csp->content_type & CT_BROTLI) +#endif +#ifdef FEATURE_ZSTD + || (csp->content_type & CT_ZSTD) #endif ) { @@ -2511,6 +2514,9 @@ static jb_err prepare_for_filtering(struct client_state *csp) csp->content_type &= ~CT_DEFLATE; #ifdef FEATURE_BROTLI csp->content_type &= ~CT_BROTLI; +#endif +#ifdef FEATURE_ZSTD + csp->content_type &= ~CT_ZSTD; #endif } } diff --git a/parsers.c b/parsers.c index 37bbc138..34b4fd02 100644 --- a/parsers.c +++ b/parsers.c @@ -67,6 +67,9 @@ #ifdef FEATURE_BROTLI #include #endif +#ifdef FEATURE_ZSTD +#include +#endif #if !defined(_WIN32) #include @@ -497,6 +500,98 @@ static jb_err decompress_iob_with_brotli(struct client_state *csp) } #endif + +#ifdef FEATURE_ZSTD +/********************************************************************* + * + * Function : decompress_iob_with_zstd + * + * Description : Decompress buffered page using zstd. + * + * Parameters : + * 1 : csp = Current client state (buffers, headers, etc...) + * + * Returns : JB_ERR_OK on success, + * JB_ERR_MEMORY if out-of-memory limit reached, and + * JB_ERR_COMPRESS if error decompressing buffer. + * + *********************************************************************/ +static jb_err decompress_iob_with_zstd(struct client_state *csp) +{ + ZSTD_DCtx *dctx; + ZSTD_inBuffer zstd_input; + ZSTD_outBuffer zstd_output; + char *decoded_buffer; + size_t result; + size_t decoded_buffer_size; + size_t encoded_size; + enum { MAX_COMPRESSION_FACTOR = 15 }; + + dctx = ZSTD_createDCtx(); + if (dctx == NULL) + { + log_error(LOG_LEVEL_ERROR, + "Failed to zstd-decompress content. ZSTD_createDCtx() failed!"); + return JB_ERR_COMPRESS; + } + + encoded_size = (size_t)(csp->iob->eod - csp->iob->cur); + decoded_buffer_size = encoded_size * MAX_COMPRESSION_FACTOR; + + if (decoded_buffer_size > csp->config->buffer_limit) + { + log_error(LOG_LEVEL_ERROR, + "Buffer limit reached before decompressing iob with zstd"); + return JB_ERR_MEMORY; + } + + decoded_buffer = malloc(decoded_buffer_size); + if (decoded_buffer == NULL) + { + log_error(LOG_LEVEL_ERROR, + "Failed to allocate %lu bytes for zstd decompression", + decoded_buffer_size); + return JB_ERR_MEMORY; + } + + zstd_input.src = csp->iob->cur; + zstd_input.size = encoded_size; + zstd_input.pos = 0; + + zstd_output.dst = decoded_buffer; + zstd_output.size = decoded_buffer_size; + zstd_output.pos = 0; + + result = ZSTD_decompressStream(dctx, &zstd_output , &zstd_input); + ZSTD_freeDCtx(dctx); + if (result == 0) + { + /* + * Update the iob, since the decompression was successful. + */ + freez(csp->iob->buf); + csp->iob->buf = decoded_buffer; + csp->iob->cur = csp->iob->buf; + csp->iob->eod = csp->iob->cur + zstd_output.pos; + csp->iob->size = decoded_buffer_size; + + log_error(LOG_LEVEL_RE_FILTER, + "Decompression successful. Old size: %lu, new size: %lu.", + encoded_size, zstd_output.pos); + + return JB_ERR_OK; + } + else + { + log_error(LOG_LEVEL_ERROR, "Failed to decompress buffer with zstd"); + freez(decoded_buffer); + + return JB_ERR_COMPRESS; + } +} +#endif + + /********************************************************************* * * Function : decompress_iob @@ -559,6 +654,13 @@ jb_err decompress_iob(struct client_state *csp) } #endif +#ifdef FEATURE_ZSTD + if (csp->content_type & CT_ZSTD) + { + return decompress_iob_with_zstd(csp); + } +#endif + if (csp->content_type & CT_GZIP) { /* @@ -2664,6 +2766,15 @@ static jb_err server_content_encoding(struct client_state *csp, char **header) csp->content_type |= CT_BROTLI; #else csp->content_type |= CT_TABOO; +#endif + } + else if (strstr(*header, "zstd")) + { +#ifdef FEATURE_ZSTD + /* Mark for zstd decompression */ + csp->content_type |= CT_ZSTD; +#else + csp->content_type |= CT_TABOO; #endif } else if (strstr(*header, "compress")) @@ -2735,6 +2846,9 @@ static jb_err server_adjust_content_encoding(struct client_state *csp, char **he && ((csp->content_type & (CT_GZIP | CT_DEFLATE)) #ifdef FEATURE_BROTLI || (csp->content_type & CT_BROTLI) +#endif +#ifdef FEATURE_ZSTD + || (csp->content_type & CT_ZSTD) #endif ) ) diff --git a/project.h b/project.h index e6009046..3c516602 100644 --- a/project.h +++ b/project.h @@ -513,13 +513,14 @@ struct iob #define CT_GZIP 0x0010U /**< gzip-compressed data. */ #define CT_DEFLATE 0x0020U /**< zlib-compressed data. */ #define CT_BROTLI 0x0040U /**< Brotli-compressed data. */ +#define CT_ZSTD 0x0080U /**< Zstandard-compressed data. */ /** * Flag to signal that the server declared the content type, * so we can differentiate between unknown and undeclared * content types. */ -#define CT_DECLARED 0x0080U +#define CT_DECLARED 0x0100U /** * The mask which includes all actions. diff --git a/templates/show-status b/templates/show-status index a6aaf8bf..7da63555 100644 --- a/templates/show-status +++ b/templates/show-status @@ -395,6 +395,12 @@ Allows to decompress gzip and zlib compressed documents for filtering. Requires external zlib library. + + FEATURE_ZSTD + @if-FEATURE_ZSTD-then@ Yes @else-not-FEATURE_ZSTD@ No @endif-FEATURE_ZSTD@ + Allows to decompress Zstandard-compressed content for filtering. + Requires external zstd library. +