X-Git-Url: http://www.privoxy.org/gitweb/?p=privoxy.git;a=blobdiff_plain;f=parsers.c;h=9b4a1298cfc80b1616e0880be55051201b032251;hp=699ae7c50f8cc4ed98afb18f047e92354dc2538e;hb=565187f94976db71fc5be074862af1ebb5afd77c;hpb=77f17a73cb0d567c22a98d9ccffc7dea0cafc6fe diff --git a/parsers.c b/parsers.c index 699ae7c5..9b4a1298 100644 --- a/parsers.c +++ b/parsers.c @@ -4,8 +4,8 @@ * * Purpose : Declares functions to parse/crunch headers and pages. * - * Copyright : Written by and Copyright (C) 2001-2017 the - * Privoxy team. http://www.privoxy.org/ + * Copyright : Written by and Copyright (C) 2001-2020 the + * Privoxy team. https://www.privoxy.org/ * * Based on the Internet Junkbuster originally written * by and Copyright (C) 1997 Anonymous Coders and @@ -64,6 +64,9 @@ #define GZIP_FLAG_COMMENT 0x10 #define GZIP_FLAG_RESERVED_BITS 0xe0 #endif +#ifdef FEATURE_BROTLI +#include +#endif #if !defined(_WIN32) && !defined(__OS2__) #include @@ -84,6 +87,9 @@ #include "list.h" #include "actions.h" #include "filters.h" +#ifdef FEATURE_HTTPS_INSPECTION +#include "ssl.h" +#endif #ifndef HAVE_STRPTIME #include "strptime.h" @@ -135,10 +141,10 @@ static jb_err server_save_content_length(struct client_state *csp, char **header static jb_err server_keep_alive(struct client_state *csp, char **header); static jb_err server_proxy_connection(struct client_state *csp, char **header); static jb_err client_keep_alive(struct client_state *csp, char **header); -static jb_err client_save_content_length(struct client_state *csp, char **header); static jb_err client_proxy_connection(struct client_state *csp, char **header); #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */ +static jb_err client_save_content_length(struct client_state *csp, char **header); static jb_err client_host_adder (struct client_state *csp); static jb_err client_xtra_adder (struct client_state *csp); static jb_err client_x_forwarded_for_adder(struct client_state *csp); @@ -182,9 +188,9 @@ static const struct parsers client_patterns[] = { { "TE:", 3, client_te }, { "Host:", 5, client_host }, { "if-modified-since:", 18, client_if_modified_since }, + { "Content-Length:", 15, client_save_content_length }, #ifdef FEATURE_CONNECTION_KEEP_ALIVE { "Keep-Alive:", 11, client_keep_alive }, - { "Content-Length:", 15, client_save_content_length }, { "Proxy-Connection:", 17, client_proxy_connection }, #else { "Keep-Alive:", 11, crumble }, @@ -378,8 +384,7 @@ jb_err add_to_iob(struct iob *iob, const size_t buffer_limit, char *src, long n) * Parameters : * 1 : iob = I/O buffer to clear. * - * Returns : JB_ERR_OK on success, JB_ERR_MEMORY if out-of-memory - * or buffer limit reached. + * Returns : N/A * *********************************************************************/ void clear_iob(struct iob *iob) @@ -390,6 +395,87 @@ void clear_iob(struct iob *iob) #ifdef FEATURE_ZLIB +#ifdef FEATURE_BROTLI +/********************************************************************* + * + * Function : decompress_iob_with_brotli + * + * Description : Decompress buffered page using Brotli. + * + * Parameters : + * 1 : csp = Current client state (buffers, headers, etc...) + * + * Returns : JB_ERR_OK on success, + * JB_ERR_MEMORY if out-of-memory limit reached, and + * JB_ERR_COMPRESS if error decompressing buffer. + * + *********************************************************************/ +static jb_err decompress_iob_with_brotli(struct client_state *csp) +{ + BrotliDecoderResult result; + char *decoded_buffer; + size_t decoded_size; + size_t decoded_buffer_size; + size_t encoded_size; + enum { MAX_COMPRESSION_FACTOR = 15 }; + + encoded_size = (size_t)(csp->iob->eod - csp->iob->cur); + /* + * The BrotliDecoderDecompress() api is a bit unfortunate + * and requires the caller to reserve enough memory for + * the decompressed content. Hopefully reserving + * MAX_COMPRESSION_FACTOR times the original size is + * sufficient. If not, BrotliDecoderDecompress() will fail. + */ + decoded_buffer_size = encoded_size * MAX_COMPRESSION_FACTOR; + + if (decoded_buffer_size > csp->config->buffer_limit) + { + log_error(LOG_LEVEL_ERROR, + "Buffer limit reached before decompressing iob with Brotli"); + return JB_ERR_MEMORY; + } + + decoded_buffer = malloc(decoded_buffer_size); + if (decoded_buffer == NULL) + { + log_error(LOG_LEVEL_ERROR, + "Failed to allocate %d bytes for Brotli decompression", + decoded_buffer_size); + return JB_ERR_MEMORY; + } + + decoded_size = decoded_buffer_size; + result = BrotliDecoderDecompress(encoded_size, + (const uint8_t *)csp->iob->cur, &decoded_size, + (uint8_t *)decoded_buffer); + if (result == BROTLI_DECODER_RESULT_SUCCESS) + { + /* + * Update the iob, since the decompression was successful. + */ + freez(csp->iob->buf); + csp->iob->buf = decoded_buffer; + csp->iob->cur = csp->iob->buf; + csp->iob->eod = csp->iob->cur + decoded_size; + csp->iob->size = decoded_buffer_size; + + log_error(LOG_LEVEL_RE_FILTER, + "Decompression successful. Old size: %d, new size: %d.", + encoded_size, decoded_size); + + return JB_ERR_OK; + } + else + { + log_error(LOG_LEVEL_ERROR, "Failed to decompress buffer with Brotli"); + freez(decoded_buffer); + + return JB_ERR_COMPRESS; + } +} +#endif + /********************************************************************* * * Function : decompress_iob @@ -445,6 +531,13 @@ jb_err decompress_iob(struct client_state *csp) return JB_ERR_COMPRESS; } +#ifdef FEATURE_BROTLI + if (csp->content_type & CT_BROTLI) + { + return decompress_iob_with_brotli(csp); + } +#endif + if (csp->content_type & CT_GZIP) { /* @@ -633,6 +726,7 @@ jb_err decompress_iob(struct client_state *csp) if (bufsize >= csp->config->buffer_limit) { log_error(LOG_LEVEL_ERROR, "Buffer limit reached while decompressing iob"); + freez(buf); return JB_ERR_MEMORY; } @@ -734,8 +828,9 @@ jb_err decompress_iob(struct client_state *csp) } else { - /* zlib thinks this is OK, so lets do the same. */ - log_error(LOG_LEVEL_INFO, "Decompression didn't result in any content."); + /* zlib thinks this is OK, so let's do the same. */ + log_error(LOG_LEVEL_RE_FILTER, + "Decompression didn't result in any content."); } } else @@ -1185,6 +1280,65 @@ jb_err sed(struct client_state *csp, int filter_server_headers) } +#ifdef FEATURE_HTTPS_INSPECTION +/********************************************************************* + * + * Function : sed_https + * + * Description : add, delete or modify lines in the HTTPS client + * header streams. Wrapper around sed(). + * + * Parameters : + * 1 : csp = Current client state (buffers, headers, etc...) + * + * Returns : JB_ERR_OK in case off success, or + * JB_ERR_MEMORY on some out-of-memory errors, or + * JB_ERR_PARSE in case of fatal parse errors. + * + *********************************************************************/ +jb_err sed_https(struct client_state *csp) +{ + jb_err err; + struct list headers; + + /* + * Temporarily replace csp->headers with csp->https_headers + * to trick sed() into filtering the https headers. + */ + headers.first = csp->headers->first; + headers.last = csp->headers->last; + csp->headers->first = csp->https_headers->first; + csp->headers->last = csp->https_headers->last; + + /* + * Start with fresh tags. Already existing tags may + * be set again. This is necessary to overrule + * URL-based patterns. + */ + destroy_list(csp->tags); + + /* + * We want client header filters and taggers + * so temporarily remove the flag. + */ + csp->flags &= ~CSP_FLAG_CLIENT_HEADER_PARSING_DONE; + err = sed(csp, FILTER_CLIENT_HEADERS); + csp->flags |= CSP_FLAG_CLIENT_HEADER_PARSING_DONE; + + /* + * Update the last header which may have changed + * due to header additions, + */ + csp->https_headers->last = csp->headers->last; + + csp->headers->first = headers.first; + csp->headers->last = headers.last; + + return err; +} +#endif /* def FEATURE_HTTPS_INSPECTION */ + + /********************************************************************* * * Function : update_server_headers @@ -1696,6 +1850,7 @@ static jb_err server_proxy_connection(struct client_state *csp, char **header) csp->flags |= CSP_FLAG_SERVER_PROXY_CONNECTION_HEADER_SET; return JB_ERR_OK; } +#endif /* def FEATURE_CONNECTION_KEEP_ALIVE */ /********************************************************************* @@ -1728,6 +1883,7 @@ static jb_err proxy_authentication(struct client_state *csp, char **header) } +#ifdef FEATURE_CONNECTION_KEEP_ALIVE /********************************************************************* * * Function : client_keep_alive @@ -1794,6 +1950,7 @@ static jb_err client_keep_alive(struct client_state *csp, char **header) return JB_ERR_OK; } +#endif /* def FEATURE_CONNECTION_KEEP_ALIVE */ /********************************************************************* @@ -1866,8 +2023,6 @@ static jb_err client_save_content_length(struct client_state *csp, char **header return JB_ERR_OK; } -#endif /* def FEATURE_CONNECTION_KEEP_ALIVE */ - /********************************************************************* @@ -1896,9 +2051,13 @@ static jb_err client_connection(struct client_state *csp, char **header) { #ifdef FEATURE_CONNECTION_KEEP_ALIVE if ((csp->config->feature_flags & RUNTIME_FEATURE_CONNECTION_SHARING) - && !(csp->flags & CSP_FLAG_SERVER_SOCKET_TAINTED)) + && !(csp->flags & CSP_FLAG_SERVER_SOCKET_TAINTED) +#ifdef FEATURE_HTTPS_INSPECTION + && !client_use_ssl(csp) +#endif + ) { - if (!strcmpic(csp->http->ver, "HTTP/1.1")) + if (!strcmpic(csp->http->version, "HTTP/1.1")) { log_error(LOG_LEVEL_HEADER, "Removing \'%s\' to imply keep-alive.", *header); @@ -2349,6 +2508,15 @@ static jb_err server_content_encoding(struct client_state *csp, char **header) /* Mark for zlib decompression */ csp->content_type |= CT_DEFLATE; } + else if (strstr(*header, "br")) + { +#ifdef FEATURE_BROTLI + /* Mark for Brotli decompression */ + csp->content_type |= CT_BROTLI; +#else + csp->content_type |= CT_TABOO; +#endif + } else if (strstr(*header, "compress")) { /* @@ -2399,7 +2567,7 @@ static jb_err server_content_encoding(struct client_state *csp, char **header) * * Description : Remove the Content-Encoding header if the * decompression was successful and the content - * has been modifed. + * has been modified. * * Parameters : * 1 : csp = Current client state (buffers, headers, etc...) @@ -2415,7 +2583,12 @@ static jb_err server_content_encoding(struct client_state *csp, char **header) static jb_err server_adjust_content_encoding(struct client_state *csp, char **header) { if ((csp->flags & CSP_FLAG_MODIFIED) - && (csp->content_type & (CT_GZIP | CT_DEFLATE))) + && ((csp->content_type & (CT_GZIP | CT_DEFLATE)) +#ifdef FEATURE_BROTLI + || (csp->content_type & CT_BROTLI) +#endif + ) + ) { /* * We successfully decompressed the content, @@ -2688,9 +2861,8 @@ static jb_err server_last_modified(struct client_state *csp, char **header) time_t now; struct tm *timeptr = NULL; long int rtime; -#ifdef HAVE_GMTIME_R struct tm gmt; -#endif + now = time(NULL); rtime = (long int)difftime(now, last_modified); if (rtime) @@ -2709,15 +2881,7 @@ static jb_err server_last_modified(struct client_state *csp, char **header) rtime *= -1; } last_modified += rtime; -#ifdef HAVE_GMTIME_R - timeptr = gmtime_r(&last_modified, &gmt); -#elif defined(MUTEX_LOCKS_AVAILABLE) - privoxy_mutex_lock(&gmtime_mutex); - timeptr = gmtime(&last_modified); - privoxy_mutex_unlock(&gmtime_mutex); -#else - timeptr = gmtime(&last_modified); -#endif + timeptr = privoxy_gmtime_r(&last_modified, &gmt); if ((NULL == timeptr) || !strftime(newheader, sizeof(newheader), "%a, %d %b %Y %H:%M:%S GMT", timeptr)) { @@ -2727,7 +2891,6 @@ static jb_err server_last_modified(struct client_state *csp, char **header) freez(*header); return JB_ERR_OK; } - freez(*header); *header = strdup("Last-Modified: "); string_append(header, newheader); @@ -3353,9 +3516,7 @@ static jb_err client_host(struct client_state *csp, char **header) static jb_err client_if_modified_since(struct client_state *csp, char **header) { char newheader[50]; -#ifdef HAVE_GMTIME_R struct tm gmt; -#endif struct tm *timeptr = NULL; time_t tm = 0; const char *newval; @@ -3413,15 +3574,7 @@ static jb_err client_if_modified_since(struct client_state *csp, char **header) *header); } tm += rtime * (negative_range ? -1 : 1); -#ifdef HAVE_GMTIME_R - timeptr = gmtime_r(&tm, &gmt); -#elif defined(MUTEX_LOCKS_AVAILABLE) - privoxy_mutex_lock(&gmtime_mutex); - timeptr = gmtime(&tm); - privoxy_mutex_unlock(&gmtime_mutex); -#else - timeptr = gmtime(&tm); -#endif + timeptr = privoxy_gmtime_r(&tm, &gmt); if ((NULL == timeptr) || !strftime(newheader, sizeof(newheader), "%a, %d %b %Y %H:%M:%S GMT", timeptr)) { @@ -3431,7 +3584,6 @@ static jb_err client_if_modified_since(struct client_state *csp, char **header) freez(*header); return JB_ERR_OK; } - freez(*header); *header = strdup("If-Modified-Since: "); string_append(header, newheader); @@ -3789,7 +3941,12 @@ static jb_err server_proxy_connection_adder(struct client_state *csp) && !(csp->flags & CSP_FLAG_SERVER_SOCKET_TAINTED) && !(csp->flags & CSP_FLAG_SERVER_PROXY_CONNECTION_HEADER_SET) && ((csp->flags & CSP_FLAG_SERVER_CONTENT_LENGTH_SET) - || (csp->flags & CSP_FLAG_CHUNKED))) + || (csp->flags & CSP_FLAG_CHUNKED)) +#ifdef FEATURE_HTTPS_INSPECTION + && !client_use_ssl(csp) + && !csp->http->ssl +#endif + ) { log_error(LOG_LEVEL_HEADER, "Adding: %s", proxy_connection_header); err = enlist(csp->headers, proxy_connection_header); @@ -3839,7 +3996,7 @@ static jb_err client_connection_header_adder(struct client_state *csp) #ifdef FEATURE_CONNECTION_KEEP_ALIVE if ((csp->config->feature_flags & RUNTIME_FEATURE_CONNECTION_KEEP_ALIVE) && !(csp->flags & CSP_FLAG_SERVER_SOCKET_TAINTED) - && !strcmpic(csp->http->ver, "HTTP/1.1")) + && !strcmpic(csp->http->version, "HTTP/1.1")) { csp->flags |= CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE; return JB_ERR_OK; @@ -3972,18 +4129,9 @@ static void add_cookie_expiry_date(char **cookie, time_t lifetime) char tmp[50]; struct tm *timeptr = NULL; time_t expiry_date = time(NULL) + lifetime; -#ifdef HAVE_GMTIME_R struct tm gmt; - timeptr = gmtime_r(&expiry_date, &gmt); -#elif defined(MUTEX_LOCKS_AVAILABLE) - privoxy_mutex_lock(&gmtime_mutex); - timeptr = gmtime(&expiry_date); - privoxy_mutex_unlock(&gmtime_mutex); -#else - timeptr = gmtime(&expiry_date); -#endif - + timeptr = privoxy_gmtime_r(&expiry_date, &gmt); if (NULL == timeptr) { log_error(LOG_LEVEL_FATAL, @@ -4326,9 +4474,10 @@ static jb_err parse_header_time(const char *header_time, time_t *result) { char recreated_date[100]; struct tm *tm; + struct tm storage; time_t result2; - tm = gmtime(result); + tm = privoxy_gmtime_r(result, &storage); if (!strftime(recreated_date, sizeof(recreated_date), time_formats[i], tm)) { @@ -4429,8 +4578,6 @@ jb_err get_destination_from_headers(const struct list *headers, struct http_requ char *p; char *host; - assert(!http->ssl); - host = get_header_value(headers, "Host:"); if (NULL == host) @@ -4483,7 +4630,88 @@ jb_err get_destination_from_headers(const struct list *headers, struct http_requ string_append(&http->cmd, " "); string_append(&http->cmd, http->url); string_append(&http->cmd, " "); - string_append(&http->cmd, http->ver); + string_append(&http->cmd, http->version); + if (http->cmd == NULL) + { + return JB_ERR_MEMORY; + } + + return JB_ERR_OK; + +} + + +#ifdef FEATURE_HTTPS_INSPECTION +/********************************************************************* + * + * Function : get_destination_from_https_headers + * + * Description : Parse the previously encrypted "Host:" header to + * get the request's destination. + * + * Parameters : + * 1 : headers = List of headers (one of them hopefully being + * the "Host:" header) + * 2 : http = storage for the result (host, port and hostport). + * + * Returns : JB_ERR_MEMORY (or terminates) in case of memory problems, + * JB_ERR_PARSE if the host header couldn't be found, + * JB_ERR_OK otherwise. + * + *********************************************************************/ +jb_err get_destination_from_https_headers(const struct list *headers, struct http_request *http) +{ + char *q; + char *p; + char *host; + + host = get_header_value(headers, "Host:"); + + if (NULL == host) + { + log_error(LOG_LEVEL_ERROR, "No \"Host:\" header found."); + return JB_ERR_PARSE; + } + + p = strdup_or_die(host); + chomp(p); + q = strdup_or_die(p); + + freez(http->hostport); + http->hostport = p; + freez(http->host); + http->host = q; + q = strchr(http->host, ':'); + if (q != NULL) + { + /* Terminate hostname and evaluate port string */ + *q++ = '\0'; + http->port = atoi(q); + } + else + { + http->port = 443; + } + + /* Rebuild request URL */ + freez(http->url); + http->url = strdup_or_die(http->path); + + log_error(LOG_LEVEL_HEADER, + "Destination extracted from \"Host\" header. New request URL: %s", + http->url); + + /* + * Regenerate request line in "proxy format" + * to make rewrites more convenient. + */ + assert(http->cmd != NULL); + freez(http->cmd); + http->cmd = strdup_or_die(http->gpc); + string_append(&http->cmd, " "); + string_append(&http->cmd, http->url); + string_append(&http->cmd, " "); + string_append(&http->cmd, http->version); if (http->cmd == NULL) { return JB_ERR_MEMORY; @@ -4492,6 +4720,7 @@ jb_err get_destination_from_headers(const struct list *headers, struct http_requ return JB_ERR_OK; } +#endif /* def FEATURE_HTTPS_INSPECTION */ /********************************************************************* @@ -4655,7 +4884,6 @@ static void create_content_length_header(unsigned long long content_length, } -#ifdef FEATURE_CONNECTION_KEEP_ALIVE /********************************************************************* * * Function : get_expected_content_length @@ -4687,7 +4915,7 @@ unsigned long long get_expected_content_length(struct list *headers) return content_length; } -#endif + /* Local Variables: