X-Git-Url: http://www.privoxy.org/gitweb/?p=privoxy.git;a=blobdiff_plain;f=parsers.c;h=14aa99ea8cbbb3a603e8501a706fd281fab97fed;hp=0beed9bf67510542c45bee97544499bef9b1729e;hb=a2f8296e90e9bc3c4e4fa50c9b42e3f12bce24ca;hpb=83dbb819f67b3919da5fea0fae6d0175def6b68a diff --git a/parsers.c b/parsers.c index 0beed9bf..14aa99ea 100644 --- a/parsers.c +++ b/parsers.c @@ -1,12 +1,11 @@ -const char parsers_rcs[] = "$Id: parsers.c,v 1.280 2013/11/24 14:24:17 fabiankeil Exp $"; /********************************************************************* * * File : $Source: /cvsroot/ijbswa/current/parsers.c,v $ * * Purpose : Declares functions to parse/crunch headers and pages. * - * Copyright : Written by and Copyright (C) 2001-2012 the - * Privoxy team. http://www.privoxy.org/ + * Copyright : Written by and Copyright (C) 2001-2021 the + * Privoxy team. https://www.privoxy.org/ * * Based on the Internet Junkbuster originally written * by and Copyright (C) 1997 Anonymous Coders and @@ -65,8 +64,11 @@ const char parsers_rcs[] = "$Id: parsers.c,v 1.280 2013/11/24 14:24:17 fabiankei #define GZIP_FLAG_COMMENT 0x10 #define GZIP_FLAG_RESERVED_BITS 0xe0 #endif +#ifdef FEATURE_BROTLI +#include +#endif -#if !defined(_WIN32) && !defined(__OS2__) +#if !defined(_WIN32) #include #endif @@ -85,17 +87,19 @@ const char parsers_rcs[] = "$Id: parsers.c,v 1.280 2013/11/24 14:24:17 fabiankei #include "list.h" #include "actions.h" #include "filters.h" +#ifdef FEATURE_HTTPS_INSPECTION +#include "ssl.h" +#endif #ifndef HAVE_STRPTIME #include "strptime.h" #endif -const char parsers_h_rcs[] = PARSERS_H_VERSION; - static char *get_header_line(struct iob *iob); static jb_err scan_headers(struct client_state *csp); static jb_err header_tagger(struct client_state *csp, char *header); static jb_err parse_header_time(const char *header_time, time_t *result); +static jb_err parse_time_header(const char *header, time_t *result); static jb_err crumble (struct client_state *csp, char **header); static jb_err filter_header (struct client_state *csp, char **header); @@ -116,6 +120,7 @@ static jb_err client_if_none_match (struct client_state *csp, char **header static jb_err crunch_client_header (struct client_state *csp, char **header); static jb_err client_x_filter (struct client_state *csp, char **header); static jb_err client_range (struct client_state *csp, char **header); +static jb_err client_expect (struct client_state *csp, char **header); static jb_err server_set_cookie (struct client_state *csp, char **header); static jb_err server_connection (struct client_state *csp, char **header); static jb_err server_content_type (struct client_state *csp, char **header); @@ -136,10 +141,10 @@ static jb_err server_save_content_length(struct client_state *csp, char **header static jb_err server_keep_alive(struct client_state *csp, char **header); static jb_err server_proxy_connection(struct client_state *csp, char **header); static jb_err client_keep_alive(struct client_state *csp, char **header); -static jb_err client_save_content_length(struct client_state *csp, char **header); static jb_err client_proxy_connection(struct client_state *csp, char **header); #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */ +static jb_err client_save_content_length(struct client_state *csp, char **header); static jb_err client_host_adder (struct client_state *csp); static jb_err client_xtra_adder (struct client_state *csp); static jb_err client_x_forwarded_for_adder(struct client_state *csp); @@ -183,9 +188,9 @@ static const struct parsers client_patterns[] = { { "TE:", 3, client_te }, { "Host:", 5, client_host }, { "if-modified-since:", 18, client_if_modified_since }, + { "Content-Length:", 15, client_save_content_length }, #ifdef FEATURE_CONNECTION_KEEP_ALIVE { "Keep-Alive:", 11, client_keep_alive }, - { "Content-Length:", 15, client_save_content_length }, { "Proxy-Connection:", 17, client_proxy_connection }, #else { "Keep-Alive:", 11, crumble }, @@ -203,6 +208,7 @@ static const struct parsers client_patterns[] = { #if 0 { "Transfer-Encoding:", 18, client_transfer_encoding }, #endif + { "Expect:", 7, client_expect }, { "*", 0, crunch_client_header }, { "*", 0, filter_header }, { NULL, 0, NULL } @@ -249,13 +255,14 @@ static const add_header_func_ptr add_server_headers[] = { /********************************************************************* * - * Function : flush_socket + * Function : flush_iob * * Description : Write any pending "buffered" content. * * Parameters : * 1 : fd = file descriptor of the socket to read * 2 : iob = The I/O buffer to flush, usually csp->iob. + * 3 : delay = Number of milliseconds to delay the writes * * Returns : On success, the number of bytes written are returned (zero * indicates nothing was written). On error, -1 is returned, @@ -265,7 +272,7 @@ static const add_header_func_ptr add_server_headers[] = { * file, the results are not portable. * *********************************************************************/ -long flush_socket(jb_socket fd, struct iob *iob) +long flush_iob(jb_socket fd, struct iob *iob, unsigned int delay) { long len = iob->eod - iob->cur; @@ -274,7 +281,7 @@ long flush_socket(jb_socket fd, struct iob *iob) return(0); } - if (write_socket(fd, iob->cur, (size_t)len)) + if (write_socket_delayed(fd, iob->cur, (size_t)len, delay)) { return(-1); } @@ -284,6 +291,27 @@ long flush_socket(jb_socket fd, struct iob *iob) } +/********************************************************************* + * + * Function : can_add_to_iob + * + * Description : Checks if the given number of bytes can be added to the given iob + * without exceeding the given buffer limit. + * + * Parameters : + * 1 : iob = Destination buffer. + * 2 : buffer_limit = Limit to which the destination may grow + * 3 : n = number of bytes to be added + * + * Returns : TRUE if the given iob can handle given number of bytes + * FALSE buffer limit will be exceeded + * + *********************************************************************/ +int can_add_to_iob(const struct iob *iob, const size_t buffer_limit, size_t n) +{ + return ((size_t)(iob->eod - iob->buf) + n + 1) > buffer_limit ? FALSE : TRUE; +} + /********************************************************************* * * Function : add_to_iob @@ -301,7 +329,7 @@ long flush_socket(jb_socket fd, struct iob *iob) * or buffer limit reached. * *********************************************************************/ -jb_err add_to_iob(struct iob *iob, const size_t buffer_limit, char *src, long n) +jb_err add_to_iob(struct iob *iob, const size_t buffer_limit, const char *src, long n) { size_t used, offset, need; char *p; @@ -319,7 +347,7 @@ jb_err add_to_iob(struct iob *iob, const size_t buffer_limit, char *src, long n) if (need > buffer_limit) { log_error(LOG_LEVEL_INFO, - "Buffer limit reached while extending the buffer (iob). Needed: %d. Limit: %d", + "Buffer limit reached while extending the buffer (iob). Needed: %lu. Limit: %lu", need, buffer_limit); return JB_ERR_MEMORY; } @@ -377,18 +405,98 @@ jb_err add_to_iob(struct iob *iob, const size_t buffer_limit, char *src, long n) * Parameters : * 1 : iob = I/O buffer to clear. * - * Returns : JB_ERR_OK on success, JB_ERR_MEMORY if out-of-memory - * or buffer limit reached. + * Returns : N/A * *********************************************************************/ void clear_iob(struct iob *iob) { free(iob->buf); - memset(iob, '\0', sizeof(*iob));; + memset(iob, '\0', sizeof(*iob)); } #ifdef FEATURE_ZLIB +#ifdef FEATURE_BROTLI +/********************************************************************* + * + * Function : decompress_iob_with_brotli + * + * Description : Decompress buffered page using Brotli. + * + * Parameters : + * 1 : csp = Current client state (buffers, headers, etc...) + * + * Returns : JB_ERR_OK on success, + * JB_ERR_MEMORY if out-of-memory limit reached, and + * JB_ERR_COMPRESS if error decompressing buffer. + * + *********************************************************************/ +static jb_err decompress_iob_with_brotli(struct client_state *csp) +{ + BrotliDecoderResult result; + char *decoded_buffer; + size_t decoded_size; + size_t decoded_buffer_size; + size_t encoded_size; + enum { MAX_COMPRESSION_FACTOR = 15 }; + + encoded_size = (size_t)(csp->iob->eod - csp->iob->cur); + /* + * The BrotliDecoderDecompress() api is a bit unfortunate + * and requires the caller to reserve enough memory for + * the decompressed content. Hopefully reserving + * MAX_COMPRESSION_FACTOR times the original size is + * sufficient. If not, BrotliDecoderDecompress() will fail. + */ + decoded_buffer_size = encoded_size * MAX_COMPRESSION_FACTOR; + + if (decoded_buffer_size > csp->config->buffer_limit) + { + log_error(LOG_LEVEL_ERROR, + "Buffer limit reached before decompressing iob with Brotli"); + return JB_ERR_MEMORY; + } + + decoded_buffer = malloc(decoded_buffer_size); + if (decoded_buffer == NULL) + { + log_error(LOG_LEVEL_ERROR, + "Failed to allocate %lu bytes for Brotli decompression", + decoded_buffer_size); + return JB_ERR_MEMORY; + } + + decoded_size = decoded_buffer_size; + result = BrotliDecoderDecompress(encoded_size, + (const uint8_t *)csp->iob->cur, &decoded_size, + (uint8_t *)decoded_buffer); + if (result == BROTLI_DECODER_RESULT_SUCCESS) + { + /* + * Update the iob, since the decompression was successful. + */ + freez(csp->iob->buf); + csp->iob->buf = decoded_buffer; + csp->iob->cur = csp->iob->buf; + csp->iob->eod = csp->iob->cur + decoded_size; + csp->iob->size = decoded_buffer_size; + + log_error(LOG_LEVEL_RE_FILTER, + "Decompression successful. Old size: %lu, new size: %lu.", + encoded_size, decoded_size); + + return JB_ERR_OK; + } + else + { + log_error(LOG_LEVEL_ERROR, "Failed to decompress buffer with Brotli"); + freez(decoded_buffer); + + return JB_ERR_COMPRESS; + } +} +#endif + /********************************************************************* * * Function : decompress_iob @@ -418,8 +526,13 @@ jb_err decompress_iob(struct client_state *csp) int status; /* return status of the inflate() call */ z_stream zstr; /* used by calls to zlib */ +#ifdef FUZZ + assert(csp->iob->cur - csp->iob->buf >= 0); + assert(csp->iob->eod - csp->iob->cur >= 0); +#else assert(csp->iob->cur - csp->iob->buf > 0); assert(csp->iob->eod - csp->iob->cur > 0); +#endif bufsize = csp->iob->size; skip_size = (size_t)(csp->iob->cur - csp->iob->buf); @@ -433,10 +546,19 @@ jb_err decompress_iob(struct client_state *csp) * This is to protect the parsing of gzipped data, * but it should(?) be valid for deflated data also. */ - log_error(LOG_LEVEL_ERROR, "Buffer too small decompressing iob"); + log_error(LOG_LEVEL_ERROR, + "Insufficient data to start decompression. Bytes in buffer: %ld", + csp->iob->eod - csp->iob->cur); return JB_ERR_COMPRESS; } +#ifdef FEATURE_BROTLI + if (csp->content_type & CT_BROTLI) + { + return decompress_iob_with_brotli(csp); + } +#endif + if (csp->content_type & CT_GZIP) { /* @@ -625,6 +747,8 @@ jb_err decompress_iob(struct client_state *csp) if (bufsize >= csp->config->buffer_limit) { log_error(LOG_LEVEL_ERROR, "Buffer limit reached while decompressing iob"); + freez(buf); + inflateEnd(&zstr); return JB_ERR_MEMORY; } @@ -643,6 +767,7 @@ jb_err decompress_iob(struct client_state *csp) { log_error(LOG_LEVEL_ERROR, "Out of memory decompressing iob"); freez(buf); + inflateEnd(&zstr); return JB_ERR_MEMORY; } else @@ -713,7 +838,7 @@ jb_err decompress_iob(struct client_state *csp) * Make sure the new uncompressed iob obeys some minimal * consistency conditions. */ - if ((csp->iob->buf < csp->iob->cur) + if ((csp->iob->buf <= csp->iob->cur) && (csp->iob->cur <= csp->iob->eod) && (csp->iob->eod <= csp->iob->buf + csp->iob->size)) { @@ -721,22 +846,20 @@ jb_err decompress_iob(struct client_state *csp) if (new_size > (size_t)0) { log_error(LOG_LEVEL_RE_FILTER, - "Decompression successful. Old size: %d, new size: %d.", + "Decompression successful. Old size: %lu, new size: %lu.", old_size, new_size); } else { - /* zlib thinks this is OK, so lets do the same. */ - log_error(LOG_LEVEL_INFO, "Decompression didn't result in any content."); + /* zlib thinks this is OK, so let's do the same. */ + log_error(LOG_LEVEL_RE_FILTER, + "Decompression didn't result in any content."); } } else { /* It seems that zlib did something weird. */ - log_error(LOG_LEVEL_ERROR, - "Unexpected error decompressing the buffer (iob): %d==%d, %d>%d, %d<%d", - csp->iob->cur, csp->iob->buf + skip_size, csp->iob->eod, csp->iob->buf, - csp->iob->eod, csp->iob->buf + csp->iob->size); + log_error(LOG_LEVEL_ERROR, "Inconsistent buffer after decompression"); return JB_ERR_COMPRESS; } @@ -750,15 +873,12 @@ jb_err decompress_iob(struct client_state *csp) * * Function : normalize_lws * - * Description : Reduces unquoted linear white space in headers - * to a single space in accordance with RFC 2616 2.2. + * Description : Reduces unquoted linear whitespace in headers to + * a single space in accordance with RFC 7230 3.2.4. * This simplifies parsing and filtering later on. * - * XXX: Remove log messages before - * the next stable release? - * * Parameters : - * 1 : header = A header with linear white space to reduce. + * 1 : header = A header with linear whitespace to reduce. * * Returns : N/A * @@ -777,7 +897,7 @@ static void normalize_lws(char *header) { q++; } - log_error(LOG_LEVEL_HEADER, "Reducing white space in '%s'", header); + log_error(LOG_LEVEL_HEADER, "Reducing whitespace in '%s'", header); string_move(p+1, q); } @@ -1091,8 +1211,8 @@ static void enforce_header_order(struct list *headers, const struct list *ordere } list_remove_all(headers); - list_duplicate(headers, new_headers); - list_remove_all(new_headers); + headers->first = new_headers->first; + headers->last = new_headers->last; return; } @@ -1116,7 +1236,8 @@ static void enforce_header_order(struct list *headers, const struct list *ordere * server and header filtering. * * Returns : JB_ERR_OK in case off success, or - * JB_ERR_MEMORY on out-of-memory error. + * JB_ERR_MEMORY on some out-of-memory errors, or + * JB_ERR_PARSE in case of fatal parse errors. * *********************************************************************/ jb_err sed(struct client_state *csp, int filter_server_headers) @@ -1142,9 +1263,9 @@ jb_err sed(struct client_state *csp, int filter_server_headers) check_negative_tag_patterns(csp, PATTERN_SPEC_NO_REQUEST_TAG_PATTERN); } - while ((err == JB_ERR_OK) && (v->str != NULL)) + while (v->str != NULL) { - for (p = csp->headers->first; (err == JB_ERR_OK) && (p != NULL); p = p->next) + for (p = csp->headers->first; p != NULL; p = p->next) { /* Header crunch()ed in previous run? -> ignore */ if (p->str == NULL) continue; @@ -1154,6 +1275,10 @@ jb_err sed(struct client_state *csp, int filter_server_headers) (v->len == CHECK_EVERY_HEADER_REMAINING)) { err = v->parser(csp, &(p->str)); + if (err != JB_ERR_OK) + { + return err; + } } } v++; @@ -1166,7 +1291,9 @@ jb_err sed(struct client_state *csp, int filter_server_headers) f++; } - if (!filter_server_headers && !list_is_empty(csp->config->ordered_client_headers)) + if (!filter_server_headers && + !list_is_empty(csp->config->ordered_client_headers) && + csp->headers->first->str != NULL) { enforce_header_order(csp->headers, csp->config->ordered_client_headers); } @@ -1175,6 +1302,67 @@ jb_err sed(struct client_state *csp, int filter_server_headers) } +#ifdef FEATURE_HTTPS_INSPECTION +/********************************************************************* + * + * Function : sed_https + * + * Description : add, delete or modify lines in the HTTPS client + * header streams. Wrapper around sed(). + * + * Parameters : + * 1 : csp = Current client state (buffers, headers, etc...) + * + * Returns : JB_ERR_OK in case off success, or + * JB_ERR_MEMORY on some out-of-memory errors, or + * JB_ERR_PARSE in case of fatal parse errors. + * + *********************************************************************/ +jb_err sed_https(struct client_state *csp) +{ + jb_err err; + struct list headers; + + /* + * Temporarily replace csp->headers with csp->https_headers + * to trick sed() into filtering the https headers. + */ + headers.first = csp->headers->first; + headers.last = csp->headers->last; + csp->headers->first = csp->https_headers->first; + csp->headers->last = csp->https_headers->last; + + /* + * Start with fresh tags. Already existing tags may + * be set again. This is necessary to overrule + * URL-based patterns. + */ + destroy_list(csp->tags); + + /* + * We want client header filters and taggers + * so temporarily remove the flag. + */ + csp->flags &= ~CSP_FLAG_CLIENT_HEADER_PARSING_DONE; + err = sed(csp, FILTER_CLIENT_HEADERS); + csp->flags |= CSP_FLAG_CLIENT_HEADER_PARSING_DONE; + + /* + * Update the https headers list which may have + * been modified due to header additions or header + * reordering. + */ + csp->https_headers->first = csp->headers->first; + csp->https_headers->last = csp->headers->last; + + csp->headers->first = headers.first; + csp->headers->last = headers.last; + + return err; +} +#endif /* def FEATURE_HTTPS_INSPECTION */ + + /********************************************************************* * * Function : update_server_headers @@ -1277,7 +1465,7 @@ jb_err update_server_headers(struct client_state *csp) *********************************************************************/ static jb_err header_tagger(struct client_state *csp, char *header) { - int wanted_filter_type; + enum filter_type wanted_filter_type; int multi_action_index; pcrs_job *job; @@ -1304,117 +1492,130 @@ static jb_err header_tagger(struct client_state *csp, char *header) return JB_ERR_OK; } - /* Execute all applying taggers */ - for (tag_name = csp->action->multi[multi_action_index]->first; - NULL != tag_name; tag_name = tag_name->next) + /* Execute all applying taggers */ + for (tag_name = csp->action->multi[multi_action_index]->first; + NULL != tag_name; tag_name = tag_name->next) + { + char *modified_tag = NULL; + char *tag = header; + size_t size = header_length; + pcrs_job *joblist; + + b = get_filter(csp, tag_name->str, wanted_filter_type); + if (b == NULL) + { + continue; + } + + joblist = b->joblist; + + if (b->dynamic) joblist = compile_dynamic_pcrs_job_list(csp, b); + + if (NULL == joblist) + { + log_error(LOG_LEVEL_TAGGING, + "Tagger %s has empty joblist. Nothing to do.", b->name); + continue; + } + + /* execute their pcrs_joblist on the header. */ + for (job = joblist; NULL != job; job = job->next) + { + const int hits = pcrs_execute(job, tag, size, &modified_tag, &size); + + if (0 < hits) { - b = get_filter(csp, tag_name->str, wanted_filter_type); - if (b != NULL) + /* Success, continue with the modified version. */ + if (tag != header) { - char *modified_tag = NULL; - char *tag = header; - size_t size = header_length; - pcrs_job *joblist = b->joblist; + freez(tag); + } + tag = modified_tag; + } + else + { + /* Tagger doesn't match */ + if (0 > hits) + { + /* Regex failure, log it but continue anyway. */ + assert(NULL != header); + log_error(LOG_LEVEL_ERROR, + "Problems with tagger \'%s\' and header \'%s\': %s", + b->name, header, pcrs_strerror(hits)); + } + freez(modified_tag); + } + } - if (b->dynamic) joblist = compile_dynamic_pcrs_job_list(csp, b); + if (b->dynamic) pcrs_free_joblist(joblist); - if (NULL == joblist) - { - log_error(LOG_LEVEL_RE_FILTER, - "Tagger %s has empty joblist. Nothing to do.", b->name); - continue; - } + /* If this tagger matched */ + if (tag != header) + { + if (0 == size) + { + /* + * There is no technical limitation which makes + * it impossible to use empty tags, but I assume + * no one would do it intentionally. + */ + freez(tag); + log_error(LOG_LEVEL_TAGGING, + "Tagger \'%s\' created an empty tag. Ignored.", b->name); + continue; + } + + if (list_contains_item(csp->action->multi[ACTION_MULTI_SUPPRESS_TAG], tag)) + { + log_error(LOG_LEVEL_TAGGING, + "Tagger \'%s\' didn't add tag \'%s\': suppressed", + b->name, tag); + freez(tag); + continue; + } - /* execute their pcrs_joblist on the header. */ - for (job = joblist; NULL != job; job = job->next) + if (!list_contains_item(csp->tags, tag)) + { + if (JB_ERR_OK != enlist(csp->tags, tag)) + { + log_error(LOG_LEVEL_ERROR, + "Insufficient memory to add tag \'%s\', " + "based on tagger \'%s\' and header \'%s\'", + tag, b->name, header); + } + else + { + char *action_message; + /* + * update the action bits right away, to make + * tagging based on tags set by earlier taggers + * of the same kind possible. + */ + if (update_action_bits_for_tag(csp, tag)) { - const int hits = pcrs_execute(job, tag, size, &modified_tag, &size); - - if (0 < hits) - { - /* Success, continue with the modified version. */ - if (tag != header) - { - freez(tag); - } - tag = modified_tag; - } - else - { - /* Tagger doesn't match */ - if (0 > hits) - { - /* Regex failure, log it but continue anyway. */ - assert(NULL != header); - log_error(LOG_LEVEL_ERROR, - "Problems with tagger \'%s\' and header \'%s\': %s", - b->name, *header, pcrs_strerror(hits)); - } - freez(modified_tag); - } + action_message = "Action bits updated accordingly."; } - - if (b->dynamic) pcrs_free_joblist(joblist); - - /* If this tagger matched */ - if (tag != header) + else { - if (0 == size) - { - /* - * There is no technical limitation which makes - * it impossible to use empty tags, but I assume - * no one would do it intentionally. - */ - freez(tag); - log_error(LOG_LEVEL_INFO, - "Tagger \'%s\' created an empty tag. Ignored.", - b->name); - continue; - } - - if (!list_contains_item(csp->tags, tag)) - { - if (JB_ERR_OK != enlist(csp->tags, tag)) - { - log_error(LOG_LEVEL_ERROR, - "Insufficient memory to add tag \'%s\', " - "based on tagger \'%s\' and header \'%s\'", - tag, b->name, *header); - } - else - { - char *action_message; - /* - * update the action bits right away, to make - * tagging based on tags set by earlier taggers - * of the same kind possible. - */ - if (update_action_bits_for_tag(csp, tag)) - { - action_message = "Action bits updated accordingly."; - } - else - { - action_message = "No action bits update necessary."; - } - - log_error(LOG_LEVEL_HEADER, - "Tagger \'%s\' added tag \'%s\'. %s", - b->name, tag, action_message); - } - } - else - { - /* XXX: Is this log-worthy? */ - log_error(LOG_LEVEL_HEADER, - "Tagger \'%s\' didn't add tag \'%s\'. " - "Tag already present", b->name, tag); - } - freez(tag); + action_message = "No action bits update necessary."; } + + log_error(LOG_LEVEL_TAGGING, + "Tagger \'%s\' added tag \'%s\'. %s", + b->name, tag, action_message); } } + else + { + /* XXX: Is this log-worthy? */ + log_error(LOG_LEVEL_TAGGING, + "Tagger \'%s\' didn't add tag \'%s\'. Tag already present", + b->name, tag); + } + freez(tag); + } + } + return JB_ERR_OK; } @@ -1451,7 +1652,7 @@ static jb_err filter_header(struct client_state *csp, char **header) struct re_filterfile_spec *b; struct list_entry *filtername; - int wanted_filter_type; + enum filter_type wanted_filter_type; int multi_action_index; if (csp->flags & CSP_FLAG_NO_FILTERING) @@ -1477,62 +1678,70 @@ static jb_err filter_header(struct client_state *csp, char **header) return JB_ERR_OK; } - /* Execute all applying header filters */ - for (filtername = csp->action->multi[multi_action_index]->first; - filtername ; filtername = filtername->next) - { - b = get_filter(csp, filtername->str, wanted_filter_type); - if (b != NULL) - { - int current_hits = 0; - pcrs_job *joblist = b->joblist; + /* Execute all applying header filters */ + for (filtername = csp->action->multi[multi_action_index]->first; + filtername != NULL; filtername = filtername->next) + { + int current_hits = 0; + pcrs_job *joblist; - if (b->dynamic) joblist = compile_dynamic_pcrs_job_list(csp, b); + b = get_filter(csp, filtername->str, wanted_filter_type); + if (b == NULL) + { + continue; + } - if (NULL == joblist) - { - log_error(LOG_LEVEL_RE_FILTER, "Filter %s has empty joblist. Nothing to do.", b->name); - continue; - } + joblist = b->joblist; - log_error(LOG_LEVEL_RE_FILTER, "filtering \'%s\' (size %d) with \'%s\' ...", - *header, size, b->name); + if (b->dynamic) joblist = compile_dynamic_pcrs_job_list(csp, b); - /* Apply all jobs from the joblist */ - for (job = joblist; NULL != job; job = job->next) - { - matches = pcrs_execute(job, *header, size, &newheader, &size); - if (0 < matches) - { - current_hits += matches; - log_error(LOG_LEVEL_HEADER, "Transforming \"%s\" to \"%s\"", *header, newheader); - freez(*header); - *header = newheader; - } - else if (0 == matches) - { - /* Filter doesn't change header */ - freez(newheader); - } - else - { - /* RegEx failure */ - log_error(LOG_LEVEL_ERROR, "Filtering \'%s\' with \'%s\' didn't work out: %s", - *header, b->name, pcrs_strerror(matches)); - if (newheader != NULL) - { - log_error(LOG_LEVEL_ERROR, "Freeing what's left: %s", newheader); - freez(newheader); - } - } - } + if (NULL == joblist) + { + log_error(LOG_LEVEL_RE_FILTER, + "Filter %s has empty joblist. Nothing to do.", b->name); + continue; + } - if (b->dynamic) pcrs_free_joblist(joblist); + log_error(LOG_LEVEL_RE_FILTER, "filtering \'%s\' (size %lu) with \'%s\' ...", + *header, size, b->name); - log_error(LOG_LEVEL_RE_FILTER, "... produced %d hits (new size %d).", current_hits, size); - hits += current_hits; + /* Apply all jobs from the joblist */ + for (job = joblist; NULL != job; job = job->next) + { + matches = pcrs_execute(job, *header, size, &newheader, &size); + if (0 < matches) + { + current_hits += matches; + log_error(LOG_LEVEL_HEADER, + "Transforming \"%s\" to \"%s\"", *header, newheader); + freez(*header); + *header = newheader; + } + else if (0 == matches) + { + /* Filter doesn't change header */ + freez(newheader); + } + else + { + /* RegEx failure */ + log_error(LOG_LEVEL_ERROR, + "Filtering \'%s\' with \'%s\' didn't work out: %s", + *header, b->name, pcrs_strerror(matches)); + if (newheader != NULL) + { + log_error(LOG_LEVEL_ERROR, "Freeing what's left: %s", newheader); + freez(newheader); } } + } + + if (b->dynamic) pcrs_free_joblist(joblist); + + log_error(LOG_LEVEL_RE_FILTER, + "... produced %d hits (new size %lu).", current_hits, size); + hits += current_hits; + } /* * Additionally checking for hits is important because if @@ -1650,6 +1859,8 @@ static jb_err server_keep_alive(struct client_state *csp, char **header) csp->flags |= CSP_FLAG_SERVER_KEEP_ALIVE_TIMEOUT_SET; } + freez(*header); + return JB_ERR_OK; } @@ -1676,6 +1887,7 @@ static jb_err server_proxy_connection(struct client_state *csp, char **header) csp->flags |= CSP_FLAG_SERVER_PROXY_CONNECTION_HEADER_SET; return JB_ERR_OK; } +#endif /* def FEATURE_CONNECTION_KEEP_ALIVE */ /********************************************************************* @@ -1708,6 +1920,7 @@ static jb_err proxy_authentication(struct client_state *csp, char **header) } +#ifdef FEATURE_CONNECTION_KEEP_ALIVE /********************************************************************* * * Function : client_keep_alive @@ -1727,7 +1940,7 @@ static jb_err proxy_authentication(struct client_state *csp, char **header) static jb_err client_keep_alive(struct client_state *csp, char **header) { unsigned int keep_alive_timeout; - const char *timeout_position = strstr(*header, ": "); + char *timeout_position; if (!(csp->config->feature_flags & RUNTIME_FEATURE_CONNECTION_KEEP_ALIVE)) { @@ -1737,31 +1950,44 @@ static jb_err client_keep_alive(struct client_state *csp, char **header) return JB_ERR_OK; } + /* Check for parameter-less format "Keep-Alive: 100" */ + timeout_position = strstr(*header, ": "); if ((NULL == timeout_position) || (1 != sscanf(timeout_position, ": %u", &keep_alive_timeout))) { - log_error(LOG_LEVEL_ERROR, "Couldn't parse: %s", *header); - } - else - { - if (keep_alive_timeout < csp->config->keep_alive_timeout) - { - log_error(LOG_LEVEL_HEADER, - "Reducing keep-alive timeout from %u to %u.", - csp->config->keep_alive_timeout, keep_alive_timeout); - csp->server_connection.keep_alive_timeout = keep_alive_timeout; - } - else + /* Assume parameter format "Keep-Alive: timeout=100" */ + timeout_position = strstr(*header, "timeout="); + if ((NULL == timeout_position) + || (1 != sscanf(timeout_position, "timeout=%u", &keep_alive_timeout))) { - /* XXX: Is this log worthy? */ log_error(LOG_LEVEL_HEADER, - "Client keep-alive timeout is %u. Sticking with %u.", - keep_alive_timeout, csp->config->keep_alive_timeout); + "Couldn't parse: '%s'. Using default timeout %u", + *header, csp->config->keep_alive_timeout); + freez(*header); + + return JB_ERR_OK; } } + if (keep_alive_timeout < csp->config->keep_alive_timeout) + { + log_error(LOG_LEVEL_HEADER, + "Reducing keep-alive timeout from %u to %u.", + csp->config->keep_alive_timeout, keep_alive_timeout); + csp->server_connection.keep_alive_timeout = keep_alive_timeout; + } + else + { + /* XXX: Is this log worthy? */ + log_error(LOG_LEVEL_HEADER, + "Client keep-alive timeout is %u. Sticking with %u.", + keep_alive_timeout, csp->config->keep_alive_timeout); + freez(*header); + } + return JB_ERR_OK; } +#endif /* def FEATURE_CONNECTION_KEEP_ALIVE */ /********************************************************************* @@ -1782,7 +2008,9 @@ static jb_err client_keep_alive(struct client_state *csp, char **header) static jb_err get_content_length(const char *header_value, unsigned long long *length) { #ifdef _WIN32 - assert(sizeof(unsigned long long) > 4); +#if SIZEOF_LONG_LONG < 8 +#error sizeof(unsigned long long) too small +#endif if (1 != sscanf(header_value, "%I64u", length)) #else if (1 != sscanf(header_value, "%llu", length)) @@ -1803,10 +2031,7 @@ static jb_err get_content_length(const char *header_value, unsigned long long *l * * Parameters : * 1 : csp = Current client state (buffers, headers, etc...) - * 2 : header = On input, pointer to header to modify. - * On output, pointer to the modified header, or NULL - * to remove the header. This function frees the - * original string if necessary. + * 2 : header = pointer to the Content-Length header * * Returns : JB_ERR_OK on success, or * JB_ERR_MEMORY on out-of-memory error. @@ -1832,8 +2057,6 @@ static jb_err client_save_content_length(struct client_state *csp, char **header return JB_ERR_OK; } -#endif /* def FEATURE_CONNECTION_KEEP_ALIVE */ - /********************************************************************* @@ -1862,9 +2085,13 @@ static jb_err client_connection(struct client_state *csp, char **header) { #ifdef FEATURE_CONNECTION_KEEP_ALIVE if ((csp->config->feature_flags & RUNTIME_FEATURE_CONNECTION_SHARING) - && !(csp->flags & CSP_FLAG_SERVER_SOCKET_TAINTED)) + && !(csp->flags & CSP_FLAG_SERVER_SOCKET_TAINTED) +#ifdef FEATURE_HTTPS_INSPECTION + && !client_use_ssl(csp) +#endif + ) { - if (!strcmpic(csp->http->ver, "HTTP/1.1")) + if (!strcmpic(csp->http->version, "HTTP/1.1")) { log_error(LOG_LEVEL_HEADER, "Removing \'%s\' to imply keep-alive.", *header); @@ -1989,6 +2216,40 @@ jb_err client_transfer_encoding(struct client_state *csp, char **header) } +/********************************************************************* + * + * Function : client_expect + * + * Description : Raise the CSP_FLAG_UNSUPPORTED_CLIENT_EXPECTATION + * if the Expect header value is unsupported. + * + * Rejecting unsupported expectations is a RFC 7231 5.1.1 + * MAY and a RFC 2616 (obsolete) MUST. + * + * Parameters : + * 1 : csp = Current client state (buffers, headers, etc...) + * 2 : header = On input, pointer to header to modify. + * On output, pointer to the modified header, or NULL + * to remove the header. This function frees the + * original string if necessary. + * + * Returns : JB_ERR_OK on success, or + * + *********************************************************************/ +jb_err client_expect(struct client_state *csp, char **header) +{ + if (0 != strcmpic(*header, "Expect: 100-continue")) + { + csp->flags |= CSP_FLAG_UNSUPPORTED_CLIENT_EXPECTATION; + log_error(LOG_LEVEL_HEADER, + "Unsupported client expectaction: %s", *header); + } + + return JB_ERR_OK; + +} + + /********************************************************************* * * Function : crumble @@ -2137,11 +2398,12 @@ static jb_err server_content_type(struct client_state *csp, char **header) */ if ((csp->content_type & CT_TEXT) || (csp->action->flags & ACTION_FORCE_TEXT_MODE)) { + jb_err err; freez(*header); *header = strdup_or_die("Content-Type: "); - string_append(header, csp->action->string[ACTION_STRING_CONTENT_TYPE]); - if (header == NULL) + err = string_append(header, csp->action->string[ACTION_STRING_CONTENT_TYPE]); + if (JB_ERR_OK != err) { log_error(LOG_LEVEL_HEADER, "Insufficient memory to replace Content-Type!"); return JB_ERR_MEMORY; @@ -2263,8 +2525,7 @@ static jb_err server_content_encoding(struct client_state *csp, char **header) /* * Log a warning if the user expects the content to be filtered. */ - if ((csp->rlist != NULL) && - (!list_is_empty(csp->action->multi[ACTION_MULTI_FILTER]))) + if (content_filters_enabled(csp->action)) { log_error(LOG_LEVEL_INFO, "SDCH-compressed content detected, content filtering disabled. " @@ -2281,6 +2542,15 @@ static jb_err server_content_encoding(struct client_state *csp, char **header) /* Mark for zlib decompression */ csp->content_type |= CT_DEFLATE; } + else if (strstr(*header, "br")) + { +#ifdef FEATURE_BROTLI + /* Mark for Brotli decompression */ + csp->content_type |= CT_BROTLI; +#else + csp->content_type |= CT_TABOO; +#endif + } else if (strstr(*header, "compress")) { /* @@ -2309,8 +2579,7 @@ static jb_err server_content_encoding(struct client_state *csp, char **header) /* * Log a warning if the user expects the content to be filtered. */ - if ((csp->rlist != NULL) && - (!list_is_empty(csp->action->multi[ACTION_MULTI_FILTER]))) + if (content_filters_enabled(csp->action)) { log_error(LOG_LEVEL_INFO, "Compressed content detected, content filtering disabled. " @@ -2332,7 +2601,7 @@ static jb_err server_content_encoding(struct client_state *csp, char **header) * * Description : Remove the Content-Encoding header if the * decompression was successful and the content - * has been modifed. + * has been modified. * * Parameters : * 1 : csp = Current client state (buffers, headers, etc...) @@ -2348,7 +2617,12 @@ static jb_err server_content_encoding(struct client_state *csp, char **header) static jb_err server_adjust_content_encoding(struct client_state *csp, char **header) { if ((csp->flags & CSP_FLAG_MODIFIED) - && (csp->content_type & (CT_GZIP | CT_DEFLATE))) + && ((csp->content_type & (CT_GZIP | CT_DEFLATE)) +#ifdef FEATURE_BROTLI + || (csp->content_type & CT_BROTLI) +#endif + ) + ) { /* * We successfully decompressed the content, @@ -2368,6 +2642,37 @@ static jb_err server_adjust_content_encoding(struct client_state *csp, char **he #endif /* defined(FEATURE_ZLIB) */ +/********************************************************************* + * + * Function : header_adjust_content_length + * + * Description : Replace given header with new Content-Length header. + * + * Parameters : + * 1 : header = On input, pointer to header to modify. + * On output, pointer to the modified header, or NULL + * to remove the header. This function frees the + * original string if necessary. + * 2 : content_length = content length value to set + * + * Returns : JB_ERR_OK on success, or + * JB_ERR_MEMORY on out-of-memory error. + * + *********************************************************************/ +jb_err header_adjust_content_length(char **header, size_t content_length) +{ + const size_t header_length = 50; + freez(*header); + *header = malloc(header_length); + if (*header == NULL) + { + return JB_ERR_MEMORY; + } + create_content_length_header(content_length, *header, header_length); + return JB_ERR_OK; +} + + /********************************************************************* * * Function : server_adjust_content_length @@ -2391,14 +2696,10 @@ static jb_err server_adjust_content_length(struct client_state *csp, char **head /* Regenerate header if the content was modified. */ if (csp->flags & CSP_FLAG_MODIFIED) { - const size_t header_length = 50; - freez(*header); - *header = malloc(header_length); - if (*header == NULL) + if (JB_ERR_OK != header_adjust_content_length(header, csp->content_length)) { return JB_ERR_MEMORY; } - create_content_length_header(csp->content_length, *header, header_length); log_error(LOG_LEVEL_HEADER, "Adjusted Content-Length to %llu", csp->content_length); } @@ -2608,13 +2909,12 @@ static jb_err server_last_modified(struct client_state *csp, char **header) } else if (0 == strcmpic(newval, "randomize")) { - const char *header_time = *header + sizeof("Last-Modified:"); - log_error(LOG_LEVEL_HEADER, "Randomizing: %s", *header); - if (JB_ERR_OK != parse_header_time(header_time, &last_modified)) + if (JB_ERR_OK != parse_time_header(*header, &last_modified)) { - log_error(LOG_LEVEL_HEADER, "Couldn't parse: %s in %s (crunching!)", header_time, *header); + log_error(LOG_LEVEL_HEADER, + "Couldn't parse time in %s (crunching!)", *header); freez(*header); } else @@ -2622,9 +2922,8 @@ static jb_err server_last_modified(struct client_state *csp, char **header) time_t now; struct tm *timeptr = NULL; long int rtime; -#ifdef HAVE_GMTIME_R struct tm gmt; -#endif + now = time(NULL); rtime = (long int)difftime(now, last_modified); if (rtime) @@ -2643,15 +2942,7 @@ static jb_err server_last_modified(struct client_state *csp, char **header) rtime *= -1; } last_modified += rtime; -#ifdef HAVE_GMTIME_R - timeptr = gmtime_r(&last_modified, &gmt); -#elif defined(MUTEX_LOCKS_AVAILABLE) - privoxy_mutex_lock(&gmtime_mutex); - timeptr = gmtime(&last_modified); - privoxy_mutex_unlock(&gmtime_mutex); -#else - timeptr = gmtime(&last_modified); -#endif + timeptr = privoxy_gmtime_r(&last_modified, &gmt); if ((NULL == timeptr) || !strftime(newheader, sizeof(newheader), "%a, %d %b %Y %H:%M:%S GMT", timeptr)) { @@ -2661,7 +2952,6 @@ static jb_err server_last_modified(struct client_state *csp, char **header) freez(*header); return JB_ERR_OK; } - freez(*header); *header = strdup("Last-Modified: "); string_append(header, newheader); @@ -2678,7 +2968,7 @@ static jb_err server_last_modified(struct client_state *csp, char **header) seconds = rtime % 60; log_error(LOG_LEVEL_HEADER, - "Randomized: %s (added %d da%s %d hou%s %d minut%s %d second%s", + "Randomized: %s (added %ld da%s %ld hou%s %ld minut%s %ld second%s", *header, days, (days == 1) ? "y" : "ys", hours, (hours == 1) ? "r" : "rs", minutes, (minutes == 1) ? "e" : "es", seconds, (seconds == 1) ? ")" : "s)"); } @@ -3225,6 +3515,13 @@ static jb_err client_host(struct client_state *csp, char **header) { char *p, *q; + if (strlen(*header) < 7) + { + log_error(LOG_LEVEL_HEADER, "Removing empty Host header"); + freez(*header); + return JB_ERR_OK; + } + if (!csp->http->hostport || (*csp->http->hostport == '*') || *csp->http->hostport == ' ' || *csp->http->hostport == '\0') { @@ -3280,9 +3577,7 @@ static jb_err client_host(struct client_state *csp, char **header) static jb_err client_if_modified_since(struct client_state *csp, char **header) { char newheader[50]; -#ifdef HAVE_GMTIME_R struct tm gmt; -#endif struct tm *timeptr = NULL; time_t tm = 0; const char *newval; @@ -3311,11 +3606,10 @@ static jb_err client_if_modified_since(struct client_state *csp, char **header) } else /* add random value */ { - const char *header_time = *header + sizeof("If-Modified-Since:"); - - if (JB_ERR_OK != parse_header_time(header_time, &tm)) + if (JB_ERR_OK != parse_time_header(*header, &tm)) { - log_error(LOG_LEVEL_HEADER, "Couldn't parse: %s in %s (crunching!)", header_time, *header); + log_error(LOG_LEVEL_HEADER, + "Couldn't parse time in %s (crunching!)", *header); freez(*header); } else @@ -3326,7 +3620,7 @@ static jb_err client_if_modified_since(struct client_state *csp, char **header) if (rtime) { - log_error(LOG_LEVEL_HEADER, "Randomizing: %s (random range: %d minut%s)", + log_error(LOG_LEVEL_HEADER, "Randomizing: %s (random range: %ld minut%s)", *header, rtime, (rtime == 1 || rtime == -1) ? "e": "es"); if (negative_range) { @@ -3337,19 +3631,11 @@ static jb_err client_if_modified_since(struct client_state *csp, char **header) } else { - log_error(LOG_LEVEL_ERROR, "Random range is 0. Assuming time transformation test.", - *header); + log_error(LOG_LEVEL_ERROR, + "Random range is 0. Assuming time transformation test."); } tm += rtime * (negative_range ? -1 : 1); -#ifdef HAVE_GMTIME_R - timeptr = gmtime_r(&tm, &gmt); -#elif defined(MUTEX_LOCKS_AVAILABLE) - privoxy_mutex_lock(&gmtime_mutex); - timeptr = gmtime(&tm); - privoxy_mutex_unlock(&gmtime_mutex); -#else - timeptr = gmtime(&tm); -#endif + timeptr = privoxy_gmtime_r(&tm, &gmt); if ((NULL == timeptr) || !strftime(newheader, sizeof(newheader), "%a, %d %b %Y %H:%M:%S GMT", timeptr)) { @@ -3359,7 +3645,6 @@ static jb_err client_if_modified_since(struct client_state *csp, char **header) freez(*header); return JB_ERR_OK; } - freez(*header); *header = strdup("If-Modified-Since: "); string_append(header, newheader); @@ -3375,7 +3660,7 @@ static jb_err client_if_modified_since(struct client_state *csp, char **header) seconds = rtime % 60; log_error(LOG_LEVEL_HEADER, - "Randomized: %s (%s %d hou%s %d minut%s %d second%s", + "Randomized: %s (%s %ld hou%s %ld minut%s %ld second%s", *header, (negative_range) ? "subtracted" : "added", hours, (hours == 1) ? "r" : "rs", minutes, (minutes == 1) ? "e" : "es", seconds, (seconds == 1) ? ")" : "s)"); @@ -3539,9 +3824,8 @@ static jb_err client_host_adder(struct client_state *csp) if (!csp->http->hostport || !*(csp->http->hostport)) { - /* XXX: When does this happen and why is it OK? */ - log_error(LOG_LEVEL_INFO, "Weirdness in client_host_adder detected and ignored."); - return JB_ERR_OK; + log_error(LOG_LEVEL_ERROR, "Destination host unknown."); + return JB_ERR_PARSE; } /* @@ -3718,7 +4002,12 @@ static jb_err server_proxy_connection_adder(struct client_state *csp) && !(csp->flags & CSP_FLAG_SERVER_SOCKET_TAINTED) && !(csp->flags & CSP_FLAG_SERVER_PROXY_CONNECTION_HEADER_SET) && ((csp->flags & CSP_FLAG_SERVER_CONTENT_LENGTH_SET) - || (csp->flags & CSP_FLAG_CHUNKED))) + || (csp->flags & CSP_FLAG_CHUNKED)) +#ifdef FEATURE_HTTPS_INSPECTION + && !client_use_ssl(csp) + && !csp->http->ssl +#endif + ) { log_error(LOG_LEVEL_HEADER, "Adding: %s", proxy_connection_header); err = enlist(csp->headers, proxy_connection_header); @@ -3734,7 +4023,8 @@ static jb_err server_proxy_connection_adder(struct client_state *csp) * Function : client_connection_header_adder * * Description : Adds a proper "Connection:" header to csp->headers - * unless the header was already present. Called from `sed'. + * unless the header was already present or it's a + * CONNECT request. Called from `sed'. * * Parameters : * 1 : csp = Current client state (buffers, headers, etc...) @@ -3753,11 +4043,21 @@ static jb_err client_connection_header_adder(struct client_state *csp) return JB_ERR_OK; } + /* + * In case of CONNECT requests "Connection: close" is implied, + * but actually setting the header has been reported to cause + * problems with some forwarding proxies that close the + * connection prematurely. + */ + if (csp->http->ssl != 0) + { + return JB_ERR_OK; + } + #ifdef FEATURE_CONNECTION_KEEP_ALIVE if ((csp->config->feature_flags & RUNTIME_FEATURE_CONNECTION_KEEP_ALIVE) && !(csp->flags & CSP_FLAG_SERVER_SOCKET_TAINTED) - && (csp->http->ssl == 0) - && !strcmpic(csp->http->ver, "HTTP/1.1")) + && !strcmpic(csp->http->version, "HTTP/1.1")) { csp->flags |= CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE; return JB_ERR_OK; @@ -3779,6 +4079,7 @@ static jb_err client_connection_header_adder(struct client_state *csp) * is a partial range (HTTP status 206) * - Rewrite HTTP/1.1 answers to HTTP/1.0 if +downgrade * action applies. + * - Normalize the HTTP-version. * * Parameters : * 1 : csp = Current client state (buffers, headers, etc...) @@ -3788,36 +4089,85 @@ static jb_err client_connection_header_adder(struct client_state *csp) * original string if necessary. * * Returns : JB_ERR_OK on success, or - * JB_ERR_MEMORY on out-of-memory error. + * JB_ERR_PARSE on fatal parse errors. * *********************************************************************/ static jb_err server_http(struct client_state *csp, char **header) { - sscanf(*header, "HTTP/%*d.%*d %d", &(csp->http->status)); - if (csp->http->status == 206) + char *reason_phrase = NULL; + char *new_response_line; + char *p; + size_t length; + unsigned int major_version; + unsigned int minor_version; + + /* Get the reason phrase which start after the second whitespace */ + p = strchr(*header, ' '); + if (NULL != p) + { + p++; + reason_phrase = strchr(p, ' '); + } + + if (reason_phrase != NULL) + { + reason_phrase++; + } + else + { + log_error(LOG_LEVEL_ERROR, + "Response line lacks reason phrase: %s", *header); + reason_phrase=""; + } + + if (3 != sscanf(*header, "HTTP/%u.%u %d", &major_version, + &minor_version, &(csp->http->status))) + { + log_error(LOG_LEVEL_ERROR, + "Failed to parse the response line: %s", *header); + return JB_ERR_PARSE; + } + + if (csp->http->status == 101 || + csp->http->status == 206) { csp->content_type = CT_TABOO; } - if ((csp->action->flags & ACTION_DOWNGRADE) != 0) + if (major_version != 1 || (minor_version != 0 && minor_version != 1)) { - /* XXX: Should we do a real validity check here? */ - if (strlen(*header) > 8) - { - (*header)[7] = '0'; - log_error(LOG_LEVEL_HEADER, "Downgraded answer to HTTP/1.0"); - } - else - { - /* - * XXX: Should we block the request or - * enlist a valid status code line here? - */ - log_error(LOG_LEVEL_INFO, "Malformed server response detected. " - "Downgrading to HTTP/1.0 impossible."); - } + /* + * According to RFC 7230 2.6 intermediaries MUST send + * their own HTTP-version in forwarded messages. + */ + log_error(LOG_LEVEL_ERROR, + "Unsupported HTTP version. Downgrading to 1.1."); + major_version = 1; + minor_version = 1; + } + + if (((csp->action->flags & ACTION_DOWNGRADE) != 0) && (minor_version == 1)) + { + log_error(LOG_LEVEL_HEADER, "Downgrading answer to HTTP/1.0"); + minor_version = 0; } + /* Rebuild response line. */ + length = sizeof("HTTP/1.1 200 ") + strlen(reason_phrase) + 1; + new_response_line = malloc_or_die(length); + + snprintf(new_response_line, length, "HTTP/%u.%u %d %s", + major_version, minor_version, csp->http->status, reason_phrase); + + if (0 != strcmp(*header, new_response_line)) + { + log_error(LOG_LEVEL_HEADER, "Response line '%s' changed to '%s'", + *header, new_response_line); + } + + freez(*header); + *header = new_response_line; + return JB_ERR_OK; } @@ -3841,18 +4191,9 @@ static void add_cookie_expiry_date(char **cookie, time_t lifetime) char tmp[50]; struct tm *timeptr = NULL; time_t expiry_date = time(NULL) + lifetime; -#ifdef HAVE_GMTIME_R struct tm gmt; - timeptr = gmtime_r(&expiry_date, &gmt); -#elif defined(MUTEX_LOCKS_AVAILABLE) - privoxy_mutex_lock(&gmtime_mutex); - timeptr = gmtime(&expiry_date); - privoxy_mutex_unlock(&gmtime_mutex); -#else - timeptr = gmtime(&expiry_date); -#endif - + timeptr = privoxy_gmtime_r(&expiry_date, &gmt); if (NULL == timeptr) { log_error(LOG_LEVEL_FATAL, @@ -4195,10 +4536,17 @@ static jb_err parse_header_time(const char *header_time, time_t *result) { char recreated_date[100]; struct tm *tm; + struct tm storage; time_t result2; - tm = gmtime(result); - strftime(recreated_date, sizeof(recreated_date), time_formats[i], tm); + tm = privoxy_gmtime_r(result, &storage); + if (!strftime(recreated_date, sizeof(recreated_date), + time_formats[i], tm)) + { + log_error(LOG_LEVEL_ERROR, "Failed to recreate date '%s' with '%s'.", + header_time, time_formats[i]); + continue; + } memset(&gmt, 0, sizeof(gmt)); if (NULL == strptime(recreated_date, time_formats[i], &gmt)) { @@ -4211,7 +4559,7 @@ static jb_err parse_header_time(const char *header_time, time_t *result) if (*result != result2) { log_error(LOG_LEVEL_ERROR, "strftime() and strptime() disagree. " - "Format: '%s'. In: '%s', out: '%s'. %d != %d. Rejecting.", + "Format: '%s'. In: '%s', out: '%s'. %ld != %ld. Rejecting.", time_formats[i], header_time, recreated_date, *result, result2); continue; } @@ -4226,6 +4574,44 @@ static jb_err parse_header_time(const char *header_time, time_t *result) } +/********************************************************************* + * + * Function : parse_time_header + * + * Description : Parses the time in an HTTP time header to get + * the numerical respresentation. + * + * Parameters : + * 1 : header = HTTP header with a time value + * 2 : result = storage for header_time in seconds + * + * Returns : JB_ERR_OK if the time format was recognized, or + * JB_ERR_PARSE otherwise. + * + *********************************************************************/ +static jb_err parse_time_header(const char *header, time_t *result) +{ + const char *header_time; + + header_time = strchr(header, ':'); + + /* + * Currently this can't happen as all callers are called + * through sed() which requires a header name followed by + * a colon. + */ + assert(header_time != NULL); + + header_time++; + if (*header_time == ' ') + { + header_time++; + } + + return parse_header_time(header_time, result); + +} + /********************************************************************* * @@ -4262,7 +4648,11 @@ jb_err get_destination_from_headers(const struct list *headers, struct http_requ return JB_ERR_PARSE; } - p = strdup_or_die(host); + p = string_tolower(host); + if (p == NULL) + { + return JB_ERR_MEMORY; + } chomp(p); q = strdup_or_die(p); @@ -4279,12 +4669,12 @@ jb_err get_destination_from_headers(const struct list *headers, struct http_requ } else { - http->port = http->ssl ? 443 : 80; + http->port = 80; } /* Rebuild request URL */ freez(http->url); - http->url = strdup(http->ssl ? "https://" : "http://"); + http->url = strdup("http://"); string_append(&http->url, http->hostport); string_append(&http->url, http->path); if (http->url == NULL) @@ -4292,14 +4682,117 @@ jb_err get_destination_from_headers(const struct list *headers, struct http_requ return JB_ERR_MEMORY; } - log_error(LOG_LEVEL_HEADER, "Destination extracted from \"Host:\" header. New request URL: %s", + log_error(LOG_LEVEL_HEADER, + "Destination extracted from \"Host\" header. New request URL: %s", http->url); + /* + * Regenerate request line in "proxy format" + * to make rewrites more convenient. + */ + assert(http->cmd != NULL); + freez(http->cmd); + http->cmd = strdup_or_die(http->gpc); + string_append(&http->cmd, " "); + string_append(&http->cmd, http->url); + string_append(&http->cmd, " "); + string_append(&http->cmd, http->version); + if (http->cmd == NULL) + { + return JB_ERR_MEMORY; + } + return JB_ERR_OK; } +#ifdef FEATURE_HTTPS_INSPECTION +/********************************************************************* + * + * Function : get_destination_from_https_headers + * + * Description : Parse the previously encrypted "Host:" header to + * get the request's destination. + * + * Parameters : + * 1 : headers = List of headers (one of them hopefully being + * the "Host:" header) + * 2 : http = storage for the result (host, port and hostport). + * + * Returns : JB_ERR_MEMORY (or terminates) in case of memory problems, + * JB_ERR_PARSE if the host header couldn't be found, + * JB_ERR_OK otherwise. + * + *********************************************************************/ +jb_err get_destination_from_https_headers(const struct list *headers, struct http_request *http) +{ + char *q; + char *p; + char *host; + + host = get_header_value(headers, "Host:"); + + if (NULL == host) + { + log_error(LOG_LEVEL_ERROR, "No \"Host:\" header found."); + return JB_ERR_PARSE; + } + + p = string_tolower(host); + if (p == NULL) + { + return JB_ERR_MEMORY; + } + chomp(p); + q = strdup_or_die(p); + + freez(http->hostport); + http->hostport = p; + freez(http->host); + http->host = q; + q = strchr(http->host, ':'); + if (q != NULL) + { + /* Terminate hostname and evaluate port string */ + *q++ = '\0'; + http->port = atoi(q); + } + else + { + http->port = 443; + } + + /* Rebuild request URL */ + freez(http->url); + http->url = strdup_or_die(http->path); + + log_error(LOG_LEVEL_HEADER, + "Destination extracted from \"Host\" header. New request URL: %s", + http->url); + + /* + * Regenerate request line in "proxy format" + * to make rewrites more convenient. + */ + assert(http->cmd != NULL); + freez(http->cmd); + http->cmd = strdup_or_die(http->gpc); + string_append(&http->cmd, " "); + string_append(&http->cmd, http->url); + string_append(&http->cmd, " "); + string_append(&http->cmd, http->version); + if (http->cmd == NULL) + { + return JB_ERR_MEMORY; + } + + return JB_ERR_OK; + +} +#endif /* def FEATURE_HTTPS_INSPECTION */ + + /********************************************************************* * * Function : create_forged_referrer @@ -4411,6 +4904,10 @@ static jb_err handle_conditional_hide_referrer_parameter(char **header, referer[hostlength+17] = '\0'; } referer_url = strstr(referer, "http://"); + if (NULL == referer_url) + { + referer_url = strstr(referer, "https://"); + } if ((NULL == referer_url) || (NULL == strstr(referer_url, host))) { /* Host has changed, Referer is invalid or a https URL. */ @@ -4450,11 +4947,17 @@ static jb_err handle_conditional_hide_referrer_parameter(char **header, static void create_content_length_header(unsigned long long content_length, char *header, size_t buffer_length) { +#ifdef _WIN32 +#if SIZEOF_LONG_LONG < 8 +#error sizeof(unsigned long long) too small +#endif + snprintf(header, buffer_length, "Content-Length: %I64u", content_length); +#else snprintf(header, buffer_length, "Content-Length: %llu", content_length); +#endif } -#ifdef FEATURE_CONNECTION_KEEP_ALIVE /********************************************************************* * * Function : get_expected_content_length @@ -4486,7 +4989,7 @@ unsigned long long get_expected_content_length(struct list *headers) return content_length; } -#endif + /* Local Variables: