Add Zstandard-decompression support

Using the reference library zstd:
https://facebook.github.io/zstd/

diff --git a/cgisimple.c b/cgisimple.c
index 961510f..58e5ae6 100644 (file)
--- a/cgisimple.c
+++ b/cgisimple.c
@@ -2202,7 +2202,15 @@ static jb_err show_defines(struct map *exports)
 #else
          0,
 #endif
-      }
+      },
+      {
+         "FEATURE_ZSTD",
+#ifdef FEATURE_ZSTD
+         1,
+#else
+         0,
+#endif
+      },
    };
 
    for (i = 0; i < SZ(features); i++)

diff --git a/configure.in b/configure.in
index 51d1b20..094ccaa 100644 (file)
--- a/configure.in
+++ b/configure.in
@@ -1317,6 +1317,28 @@ if test X"$WITH_BROTLI" != Xno; then
 fi
 
 
+dnl ========================================================
+dnl Check for zstd which can be used for decompression
+dnl ========================================================
+WITH_ZSTD=no
+AC_ARG_WITH(zstd,
+AC_HELP_STRING([--with-zstd], [Enable zstd detection])
+AC_HELP_STRING([--without-zstd], [Disable zstd detection]),
+  WITH_ZSTD=$withval)
+
+if test X"$WITH_ZSTD" != Xno; then
+
+  LIBS="$LIBS -lzstd"
+  AC_CHECK_LIB(zstd, ZSTD_decompressStream)
+
+  AC_CHECK_HEADERS(zstd.h,
+    FEATURE_ZSTD=1
+    AC_DEFINE(FEATURE_ZSTD, 1, [If zstd is used for decompression])
+    AC_SUBST(FEATURE_ZSTD, [1])
+  )
+fi
+
+
 dnl =================================================================
 dnl Final cleanup and output
 dnl =================================================================

diff --git a/filters.c b/filters.c
index 7f4a3e4..8daa8c5 100644 (file)
--- a/filters.c
+++ b/filters.c
@@ -2485,6 +2485,9 @@ static jb_err prepare_for_filtering(struct client_state *csp)
    if ((csp->content_type & (CT_GZIP|CT_DEFLATE))
 #ifdef FEATURE_BROTLI
       || (csp->content_type & CT_BROTLI)
+#endif
+#ifdef FEATURE_ZSTD
+      || (csp->content_type & CT_ZSTD)
 #endif
        )
    {
@@ -2511,6 +2514,9 @@ static jb_err prepare_for_filtering(struct client_state *csp)
          csp->content_type &= ~CT_DEFLATE;
 #ifdef FEATURE_BROTLI
          csp->content_type &= ~CT_BROTLI;
+#endif
+#ifdef FEATURE_ZSTD
+         csp->content_type &= ~CT_ZSTD;
 #endif
       }
    }

diff --git a/parsers.c b/parsers.c
index 37bbc13..34b4fd0 100644 (file)
--- a/parsers.c
+++ b/parsers.c
@@ -67,6 +67,9 @@
 #ifdef FEATURE_BROTLI
 #include <brotli/decode.h>
 #endif
+#ifdef FEATURE_ZSTD
+#include <zstd.h>
+#endif
 
 #if !defined(_WIN32)
 #include <unistd.h>
@@ -497,6 +500,98 @@ static jb_err decompress_iob_with_brotli(struct client_state *csp)
 }
 #endif
 
+
+#ifdef FEATURE_ZSTD
+/*********************************************************************
+ *
+ * Function    :  decompress_iob_with_zstd
+ *
+ * Description :  Decompress buffered page using zstd.
+ *
+ * Parameters  :
+ *          1  :  csp = Current client state (buffers, headers, etc...)
+ *
+ * Returns     :  JB_ERR_OK on success,
+ *                JB_ERR_MEMORY if out-of-memory limit reached, and
+ *                JB_ERR_COMPRESS if error decompressing buffer.
+ *
+ *********************************************************************/
+static jb_err decompress_iob_with_zstd(struct client_state *csp)
+{
+   ZSTD_DCtx *dctx;
+   ZSTD_inBuffer zstd_input;
+   ZSTD_outBuffer zstd_output;
+   char *decoded_buffer;
+   size_t result;
+   size_t decoded_buffer_size;
+   size_t encoded_size;
+   enum { MAX_COMPRESSION_FACTOR = 15 };
+
+   dctx = ZSTD_createDCtx();
+   if (dctx == NULL)
+   {
+      log_error(LOG_LEVEL_ERROR,
+         "Failed to zstd-decompress content. ZSTD_createDCtx() failed!");
+      return JB_ERR_COMPRESS;
+   }
+
+   encoded_size = (size_t)(csp->iob->eod - csp->iob->cur);
+   decoded_buffer_size = encoded_size * MAX_COMPRESSION_FACTOR;
+
+   if (decoded_buffer_size > csp->config->buffer_limit)
+   {
+      log_error(LOG_LEVEL_ERROR,
+         "Buffer limit reached before decompressing iob with zstd");
+      return JB_ERR_MEMORY;
+   }
+
+   decoded_buffer = malloc(decoded_buffer_size);
+   if (decoded_buffer == NULL)
+   {
+      log_error(LOG_LEVEL_ERROR,
+         "Failed to allocate %lu bytes for zstd decompression",
+         decoded_buffer_size);
+      return JB_ERR_MEMORY;
+   }
+
+   zstd_input.src = csp->iob->cur;
+   zstd_input.size = encoded_size;
+   zstd_input.pos = 0;
+
+   zstd_output.dst = decoded_buffer;
+   zstd_output.size = decoded_buffer_size;
+   zstd_output.pos = 0;
+
+   result = ZSTD_decompressStream(dctx, &zstd_output , &zstd_input);
+   ZSTD_freeDCtx(dctx);
+   if (result == 0)
+   {
+      /*
+       * Update the iob, since the decompression was successful.
+       */
+      freez(csp->iob->buf);
+      csp->iob->buf  = decoded_buffer;
+      csp->iob->cur  = csp->iob->buf;
+      csp->iob->eod  = csp->iob->cur + zstd_output.pos;
+      csp->iob->size = decoded_buffer_size;
+
+      log_error(LOG_LEVEL_RE_FILTER,
+         "Decompression successful. Old size: %lu, new size: %lu.",
+         encoded_size, zstd_output.pos);
+
+      return JB_ERR_OK;
+   }
+   else
+   {
+      log_error(LOG_LEVEL_ERROR, "Failed to decompress buffer with zstd");
+      freez(decoded_buffer);
+
+      return JB_ERR_COMPRESS;
+   }
+}
+#endif
+
+
 /*********************************************************************
  *
  * Function    :  decompress_iob
@@ -559,6 +654,13 @@ jb_err decompress_iob(struct client_state *csp)
    }
 #endif
 
+#ifdef FEATURE_ZSTD
+   if (csp->content_type & CT_ZSTD)
+   {
+      return decompress_iob_with_zstd(csp);
+   }
+#endif
+
    if (csp->content_type & CT_GZIP)
    {
       /*
@@ -2664,6 +2766,15 @@ static jb_err server_content_encoding(struct client_state *csp, char **header)
       csp->content_type |= CT_BROTLI;
 #else
       csp->content_type |= CT_TABOO;
+#endif
+   }
+   else if (strstr(*header, "zstd"))
+   {
+#ifdef FEATURE_ZSTD
+      /* Mark for zstd decompression */
+      csp->content_type |= CT_ZSTD;
+#else
+      csp->content_type |= CT_TABOO;
 #endif
    }
    else if (strstr(*header, "compress"))
@@ -2735,6 +2846,9 @@ static jb_err server_adjust_content_encoding(struct client_state *csp, char **he
       && ((csp->content_type & (CT_GZIP | CT_DEFLATE))
 #ifdef FEATURE_BROTLI
          || (csp->content_type & CT_BROTLI)
+#endif
+#ifdef FEATURE_ZSTD
+         || (csp->content_type & CT_ZSTD)
 #endif
          )
       )

diff --git a/project.h b/project.h
index e600904..3c51660 100644 (file)
--- a/project.h
+++ b/project.h
@@ -513,13 +513,14 @@ struct iob
 #define CT_GZIP    0x0010U /**< gzip-compressed data. */
 #define CT_DEFLATE 0x0020U /**< zlib-compressed data. */
 #define CT_BROTLI  0x0040U /**< Brotli-compressed data. */
+#define CT_ZSTD    0x0080U /**< Zstandard-compressed data. */
 
 /**
  * Flag to signal that the server declared the content type,
  * so we can differentiate between unknown and undeclared
  * content types.
  */
-#define CT_DECLARED 0x0080U
+#define CT_DECLARED 0x0100U
 
 /**
  * The mask which includes all actions.

diff --git a/templates/show-status b/templates/show-status
index a6aaf8b..7da6355 100644 (file)
--- a/templates/show-status
+++ b/templates/show-status
@@ -395,6 +395,12 @@
               <td>Allows to decompress gzip and zlib compressed documents for filtering.
                 Requires external zlib library.</td>
             </tr>
+            <tr>
+              <td><code>FEATURE_ZSTD</code></td>
+              <td>@if-FEATURE_ZSTD-then@ Yes @else-not-FEATURE_ZSTD@ No @endif-FEATURE_ZSTD@</td>
+              <td>Allows to decompress Zstandard-compressed content for filtering.
+                Requires external zstd library.</td>
+            </tr>
           </table>
       </td>
     </tr>