X-Git-Url: http://www.privoxy.org/gitweb/?p=privoxy.git;a=blobdiff_plain;f=parsers.c;h=51ff6944f14797c1e53a9a1510634b9472c8ab03;hp=a8d368380318e43b1830ce11c642c5a064328168;hb=83a4ca758c4dccd313f23f9d736f43c34e16e08a;hpb=63728b12fd2e66db3f948a2ef5422b007825944c diff --git a/parsers.c b/parsers.c index a8d36838..51ff6944 100644 --- a/parsers.c +++ b/parsers.c @@ -1,21 +1,9 @@ -const char parsers_rcs[] = "$Id: parsers.c,v 1.185 2009/06/27 11:25:33 fabiankeil Exp $"; +const char parsers_rcs[] = "$Id: parsers.c,v 1.237 2011/11/06 11:36:27 fabiankeil Exp $"; /********************************************************************* * * File : $Source: /cvsroot/ijbswa/current/parsers.c,v $ * * Purpose : Declares functions to parse/crunch headers and pages. - * Functions declared include: - * `add_to_iob', `client_cookie_adder', `client_from', - * `client_referrer', `client_send_cookie', `client_ua', - * `client_uagent', `client_x_forwarded', - * `client_x_forwarded_adder', `client_xtra_adder', - * `content_type', `crumble', `destroy_list', `enlist', - * `flush_socket', ``get_header', `sed', `filter_header' - * `server_content_encoding', `server_content_disposition', - * `server_last_modified', `client_accept_language', - * `crunch_client_header', `client_if_modified_since', - * `client_if_none_match', `get_destination_from_headers', - * `parse_header_time', `decompress_iob' and `server_set_cookie'. * * Copyright : Written by and Copyright (C) 2001-2009 the * Privoxy team. http://www.privoxy.org/ @@ -152,10 +140,16 @@ static jb_err server_http (struct client_state *csp, char **header static jb_err crunch_server_header (struct client_state *csp, char **header); static jb_err server_last_modified (struct client_state *csp, char **header); static jb_err server_content_disposition(struct client_state *csp, char **header); +#ifdef FEATURE_ZLIB +static jb_err server_adjust_content_encoding(struct client_state *csp, char **header); +#endif #ifdef FEATURE_CONNECTION_KEEP_ALIVE static jb_err server_save_content_length(struct client_state *csp, char **header); static jb_err server_keep_alive(struct client_state *csp, char **header); +static jb_err server_proxy_connection(struct client_state *csp, char **header); +static jb_err client_keep_alive(struct client_state *csp, char **header); +static jb_err client_save_content_length(struct client_state *csp, char **header); #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */ static jb_err client_host_adder (struct client_state *csp); @@ -171,7 +165,6 @@ static jb_err create_forged_referrer(char **header, const char *hostport); static jb_err create_fake_referrer(char **header, const char *fake_referrer); static jb_err handle_conditional_hide_referrer_parameter(char **header, const char *host, const int parameter_conditional_block); -static const char *get_appropiate_connection_header(const struct client_state *csp); static void create_content_length_header(unsigned long long content_length, char *header, size_t buffer_length); @@ -182,10 +175,10 @@ struct parsers { /** The header prefix to match */ const char *str; - + /** The length of the prefix to match */ const size_t len; - + /** The function to apply to this line */ const parser_func_ptr parser; }; @@ -201,7 +194,10 @@ static const struct parsers client_patterns[] = { { "TE:", 3, client_te }, { "Host:", 5, client_host }, { "if-modified-since:", 18, client_if_modified_since }, -#ifndef FEATURE_CONNECTION_KEEP_ALIVE +#ifdef FEATURE_CONNECTION_KEEP_ALIVE + { "Keep-Alive:", 11, client_keep_alive }, + { "Content-Length:", 15, client_save_content_length }, +#else { "Keep-Alive:", 11, crumble }, #endif { "connection:", 11, client_connection }, @@ -228,6 +224,7 @@ static const struct parsers server_patterns[] = { #ifdef FEATURE_CONNECTION_KEEP_ALIVE { "Content-Length:", 15, server_save_content_length }, { "Keep-Alive:", 11, server_keep_alive }, + { "Proxy-Connection:", 17, server_proxy_connection }, #else { "Keep-Alive:", 11, crumble }, #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */ @@ -243,7 +240,6 @@ static const add_header_func_ptr add_client_headers[] = { client_host_adder, client_x_forwarded_for_adder, client_xtra_adder, - /* Temporarily disabled: client_accept_encoding_adder, */ client_connection_header_adder, NULL }; @@ -312,7 +308,7 @@ long flush_socket(jb_socket fd, struct iob *iob) jb_err add_to_iob(struct client_state *csp, char *buf, long n) { struct iob *iob = csp->iob; - size_t used, offset, need, want; + size_t used, offset, need; char *p; if (n <= 0) return JB_ERR_OK; @@ -335,8 +331,13 @@ jb_err add_to_iob(struct client_state *csp, char *buf, long n) if (need > iob->size) { - for (want = csp->iob->size ? csp->iob->size : 512; want <= need;) want *= 2; - + size_t want = csp->iob->size ? csp->iob->size : 512; + + while (want <= need) + { + want *= 2; + } + if (want <= csp->config->buffer_limit && NULL != (p = (char *)realloc(iob->buf, want))) { iob->size = want; @@ -392,7 +393,7 @@ jb_err add_to_iob(struct client_state *csp, char *buf, long n) jb_err decompress_iob(struct client_state *csp) { char *buf; /* new, uncompressed buffer */ - char *cur; /* Current iob position (to keep the original + char *cur; /* Current iob position (to keep the original * iob->cur unmodified if we return early) */ size_t bufsize; /* allocated size of the new buffer */ size_t old_size; /* Content size before decompression */ @@ -540,7 +541,7 @@ jb_err decompress_iob(struct client_state *csp) * * Fortunately, add_to_iob() has thoughtfully null-terminated * the buffer; we can just increment the end pointer to include - * the dummy byte. + * the dummy byte. */ csp->iob->eod++; } @@ -582,7 +583,7 @@ jb_err decompress_iob(struct client_state *csp) assert(bufsize >= skip_size); memcpy(buf, csp->iob->buf, skip_size); - zstr.avail_out = bufsize - skip_size; + zstr.avail_out = (uInt)(bufsize - skip_size); zstr.next_out = (Bytef *)buf + skip_size; /* Try to decompress the whole stream in one shot. */ @@ -605,10 +606,12 @@ jb_err decompress_iob(struct client_state *csp) } /* - * If we tried the limit and still didn't have enough - * memory, just give up. + * If we reached the buffer limit and still didn't have enough + * memory, just give up. Due to the ceiling enforced by the next + * if block we could actually check for equality here, but as it + * can be easily mistaken for a bug we don't. */ - if (bufsize == csp->config->buffer_limit) + if (bufsize >= csp->config->buffer_limit) { log_error(LOG_LEVEL_ERROR, "Buffer limit reached while decompressing iob"); return JB_ERR_MEMORY; @@ -622,7 +625,7 @@ jb_err decompress_iob(struct client_state *csp) { bufsize = csp->config->buffer_limit; } - + /* Try to allocate the new buffer. */ tmpbuf = realloc(buf, bufsize); if (NULL == tmpbuf) @@ -640,7 +643,7 @@ jb_err decompress_iob(struct client_state *csp) * buffer, which may be in a location different from * the old one. */ - zstr.avail_out += bufsize - oldbufsize; + zstr.avail_out += (uInt)(bufsize - oldbufsize); zstr.next_out = (Bytef *)tmpbuf + bufsize - zstr.avail_out; /* @@ -649,7 +652,6 @@ jb_err decompress_iob(struct client_state *csp) */ assert(zstr.avail_out == tmpbuf + bufsize - (char *)zstr.next_out); assert((char *)zstr.next_out == tmpbuf + ((char *)oldnext_out - buf)); - assert(zstr.avail_out > 0U); buf = tmpbuf; } @@ -695,7 +697,7 @@ jb_err decompress_iob(struct client_state *csp) csp->iob->cur = csp->iob->buf + skip_size; csp->iob->eod = (char *)zstr.next_out; csp->iob->size = bufsize; - + /* * Make sure the new uncompressed iob obeys some minimal * consistency conditions. @@ -867,7 +869,7 @@ char *get_header(struct iob *iob) * Header spans multiple lines, append the next one. */ char *continued_header; - + continued_header = get_header_line(iob); if ((continued_header == NULL) || (*continued_header == '\0')) { @@ -999,7 +1001,7 @@ char *get_header_value(const struct list *header_list, const char *header_name) } } - /* + /* * Not found */ return NULL; @@ -1011,7 +1013,7 @@ char *get_header_value(const struct list *header_list, const char *header_name) * * Function : scan_headers * - * Description : Scans headers, applies tags and updates action bits. + * Description : Scans headers, applies tags and updates action bits. * * Parameters : * 1 : csp = Current client state (buffers, headers, etc...) @@ -1127,7 +1129,7 @@ jb_err update_server_headers(struct client_state *csp) { "Content-Length:", 15, server_adjust_content_length }, { "Transfer-Encoding:", 18, server_transfer_coding }, #ifdef FEATURE_ZLIB - { "Content-Encoding:", 17, server_content_encoding }, + { "Content-Encoding:", 17, server_adjust_content_encoding }, #endif /* def FEATURE_ZLIB */ { NULL, 0, NULL } }; @@ -1172,6 +1174,18 @@ jb_err update_server_headers(struct client_state *csp) } #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */ +#ifdef FEATURE_COMPRESSION + if ((JB_ERR_OK == err) + && (csp->flags & CSP_FLAG_BUFFERED_CONTENT_DEFLATED)) + { + err = enlist_unique_header(csp->headers, "Content-Encoding", "deflate"); + if (JB_ERR_OK == err) + { + log_error(LOG_LEVEL_HEADER, "Added header: Content-Encoding: deflate"); + } + } +#endif + return err; } @@ -1204,7 +1218,6 @@ static jb_err header_tagger(struct client_state *csp, char *header) struct re_filterfile_spec *b; struct list_entry *tag_name; - int found_filters = 0; const size_t header_length = strlen(header); if (csp->flags & CSP_FLAG_CLIENT_HEADER_PARSING_DONE) @@ -1218,21 +1231,7 @@ static jb_err header_tagger(struct client_state *csp, char *header) multi_action_index = ACTION_MULTI_CLIENT_HEADER_TAGGER; } - /* Check if there are any filters */ - for (i = 0; i < MAX_AF_FILES; i++) - { - fl = csp->rlist[i]; - if (NULL != fl) - { - if (NULL != fl->f) - { - found_filters = 1; - break; - } - } - } - - if (0 == found_filters) + if (filters_available(csp) == FALSE) { log_error(LOG_LEVEL_ERROR, "Inconsistent configuration: " "tagging enabled, but no taggers available."); @@ -1332,7 +1331,7 @@ static jb_err header_tagger(struct client_state *csp, char *header) b->name); continue; } - + if (!list_contains_item(csp->tags, tag)) { if (JB_ERR_OK != enlist(csp->tags, tag)) @@ -1415,7 +1414,7 @@ static jb_err filter_header(struct client_state *csp, char **header) struct re_filterfile_spec *b; struct list_entry *filtername; - int i, found_filters = 0; + int i; int wanted_filter_type; int multi_action_index; @@ -1435,23 +1434,7 @@ static jb_err filter_header(struct client_state *csp, char **header) multi_action_index = ACTION_MULTI_CLIENT_HEADER_FILTER; } - /* - * Need to check the set of re_filterfiles... - */ - for (i = 0; i < MAX_AF_FILES; i++) - { - fl = csp->rlist[i]; - if (NULL != fl) - { - if (NULL != fl->f) - { - found_filters = 1; - break; - } - } - } - - if (0 == found_filters) + if (filters_available(csp) == FALSE) { log_error(LOG_LEVEL_ERROR, "Inconsistent configuration: " "header filtering enabled, but no matching filters available."); @@ -1512,7 +1495,7 @@ static jb_err filter_header(struct client_state *csp, char **header) matches = pcrs_execute(job, *header, size, &newheader, &size); if ( 0 < matches ) { - current_hits += matches; + current_hits += matches; log_error(LOG_LEVEL_HEADER, "Transforming \"%s\" to \"%s\"", *header, newheader); freez(*header); *header = newheader; @@ -1624,7 +1607,7 @@ static jb_err server_connection(struct client_state *csp, char **header) * * Function : server_keep_alive * - * Description : Stores the servers keep alive timeout. + * Description : Stores the server's keep alive timeout. * * Parameters : * 1 : csp = Current client state (buffers, headers, etc...) @@ -1662,6 +1645,157 @@ static jb_err server_keep_alive(struct client_state *csp, char **header) "Server keep-alive timeout is %u. Sticking with %u.", keep_alive_timeout, csp->server_connection.keep_alive_timeout); } + csp->flags |= CSP_FLAG_SERVER_KEEP_ALIVE_TIMEOUT_SET; + } + + return JB_ERR_OK; +} + + +/********************************************************************* + * + * Function : server_proxy_connection + * + * Description : Figures out whether or not we should add a + * Proxy-Connection header. + * + * Parameters : + * 1 : csp = Current client state (buffers, headers, etc...) + * 2 : header = On input, pointer to header to modify. + * On output, pointer to the modified header, or NULL + * to remove the header. This function frees the + * original string if necessary. + * + * Returns : JB_ERR_OK. + * + *********************************************************************/ +static jb_err server_proxy_connection(struct client_state *csp, char **header) +{ + csp->flags |= CSP_FLAG_SERVER_PROXY_CONNECTION_HEADER_SET; + return JB_ERR_OK; +} + + +/********************************************************************* + * + * Function : client_keep_alive + * + * Description : Stores the client's keep alive timeout. + * + * Parameters : + * 1 : csp = Current client state (buffers, headers, etc...) + * 2 : header = On input, pointer to header to modify. + * On output, pointer to the modified header, or NULL + * to remove the header. This function frees the + * original string if necessary. + * + * Returns : JB_ERR_OK. + * + *********************************************************************/ +static jb_err client_keep_alive(struct client_state *csp, char **header) +{ + unsigned int keep_alive_timeout; + const char *timeout_position = strstr(*header, ": "); + + if (!(csp->config->feature_flags & RUNTIME_FEATURE_CONNECTION_KEEP_ALIVE)) + { + log_error(LOG_LEVEL_HEADER, + "keep-alive support is disabled. Crunching: %s.", *header); + freez(*header); + return JB_ERR_OK; + } + + if ((NULL == timeout_position) + || (1 != sscanf(timeout_position, ": %u", &keep_alive_timeout))) + { + log_error(LOG_LEVEL_ERROR, "Couldn't parse: %s", *header); + } + else + { + if (keep_alive_timeout < csp->config->keep_alive_timeout) + { + log_error(LOG_LEVEL_HEADER, + "Reducing keep-alive timeout from %u to %u.", + csp->config->keep_alive_timeout, keep_alive_timeout); + csp->server_connection.keep_alive_timeout = keep_alive_timeout; + } + else + { + /* XXX: Is this log worthy? */ + log_error(LOG_LEVEL_HEADER, + "Client keep-alive timeout is %u. Sticking with %u.", + keep_alive_timeout, csp->config->keep_alive_timeout); + } + } + + return JB_ERR_OK; +} + + +/********************************************************************* + * + * Function : get_content_length + * + * Description : Gets the content length specified in a + * Content-Length header. + * + * Parameters : + * 1 : header = The Content-Length header. + * 2 : length = Storage to return the value. + * + * Returns : JB_ERR_OK on success, or + * JB_ERR_PARSE if no value is recognized. + * + *********************************************************************/ +static jb_err get_content_length(const char *header, unsigned long long *length) +{ + assert(header[14] == ':'); + +#ifdef _WIN32 + assert(sizeof(unsigned long long) > 4); + if (1 != sscanf(header+14, ": %I64u", length)) +#else + if (1 != sscanf(header+14, ": %llu", length)) +#endif + { + return JB_ERR_PARSE; + } + + return JB_ERR_OK; +} + + +/********************************************************************* + * + * Function : client_save_content_length + * + * Description : Save the Content-Length sent by the client. + * + * Parameters : + * 1 : csp = Current client state (buffers, headers, etc...) + * 2 : header = On input, pointer to header to modify. + * On output, pointer to the modified header, or NULL + * to remove the header. This function frees the + * original string if necessary. + * + * Returns : JB_ERR_OK on success, or + * JB_ERR_MEMORY on out-of-memory error. + * + *********************************************************************/ +static jb_err client_save_content_length(struct client_state *csp, char **header) +{ + unsigned long long content_length = 0; + + assert(*(*header+14) == ':'); + + if (JB_ERR_OK != get_content_length(*header, &content_length)) + { + log_error(LOG_LEVEL_ERROR, "Crunching invalid header: %s", *header); + freez(*header); + } + else + { + csp->expected_client_content_length = content_length; } return JB_ERR_OK; @@ -1675,7 +1809,7 @@ static jb_err server_keep_alive(struct client_state *csp, char **header) * Function : client_connection * * Description : Makes sure a proper "Connection:" header is - * set and signals connection_header_adder + * set and signals connection_header_adder * to do nothing. * * Parameters : @@ -1691,43 +1825,69 @@ static jb_err server_keep_alive(struct client_state *csp, char **header) *********************************************************************/ static jb_err client_connection(struct client_state *csp, char **header) { - const char *wanted_header = get_appropiate_connection_header(csp); + static const char connection_close[] = "Connection: close"; - if (strcmpic(*header, wanted_header)) + if (!strcmpic(*header, connection_close)) { #ifdef FEATURE_CONNECTION_KEEP_ALIVE - if (!(csp->config->feature_flags & RUNTIME_FEATURE_CONNECTION_SHARING)) + if ((csp->config->feature_flags & RUNTIME_FEATURE_CONNECTION_SHARING)) { - log_error(LOG_LEVEL_HEADER, - "Keeping the client header '%s' around. " - "The connection will not be kept alive.", - *header); + if (!strcmpic(csp->http->ver, "HTTP/1.1")) + { + log_error(LOG_LEVEL_HEADER, + "Removing \'%s\' to imply keep-alive.", *header); + freez(*header); + /* + * While we imply keep-alive to the server, + * we have to remember that the client didn't. + */ + csp->flags &= ~CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE; + } + else + { + char *old_header = *header; + + *header = strdup("Connection: keep-alive"); + if (header == NULL) + { + return JB_ERR_MEMORY; + } + log_error(LOG_LEVEL_HEADER, + "Replaced: \'%s\' with \'%s\'", old_header, *header); + freez(old_header); + } } else -#endif /* def FEATURE_CONNECTION_KEEP_ALIVE */ { - char *old_header = *header; - - *header = strdup(wanted_header); - if (header == NULL) - { - return JB_ERR_MEMORY; - } log_error(LOG_LEVEL_HEADER, - "Replaced: \'%s\' with \'%s\'", old_header, *header); - freez(old_header); + "Keeping the client header '%s' around. " + "The connection will not be kept alive.", + *header); + csp->flags &= ~CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE; } } -#ifdef FEATURE_CONNECTION_KEEP_ALIVE - else + else if ((csp->config->feature_flags & RUNTIME_FEATURE_CONNECTION_KEEP_ALIVE)) { log_error(LOG_LEVEL_HEADER, "Keeping the client header '%s' around. " "The server connection will be kept alive if possible.", *header); csp->flags |= CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE; - } #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */ + } + else + { + char *old_header = *header; + + *header = strdup(connection_close); + if (header == NULL) + { + return JB_ERR_MEMORY; + } + log_error(LOG_LEVEL_HEADER, + "Replaced: \'%s\' with \'%s\'", old_header, *header); + freez(old_header); + } /* Signal client_connection_adder() to return early. */ csp->flags |= CSP_FLAG_CLIENT_CONNECTION_HEADER_SET; @@ -1791,7 +1951,7 @@ static jb_err crunch_server_header(struct client_state *csp, char **header) /* Is the current header the lucky one? */ if (strstr(*header, crunch_pattern)) { - log_error(LOG_LEVEL_HEADER, "Crunching server header: %s (contains: %s)", *header, crunch_pattern); + log_error(LOG_LEVEL_HEADER, "Crunching server header: %s (contains: %s)", *header, crunch_pattern); freez(*header); } } @@ -1805,7 +1965,7 @@ static jb_err crunch_server_header(struct client_state *csp, char **header) * Function : server_content_type * * Description : Set the content-type for filterable types (text/.*, - * .*xml.*, javascript and image/gif) unless filtering has been + * .*xml.*, .*script.* and image/gif) unless filtering has been * forbidden (CT_TABOO) while parsing earlier headers. * NOTE: Since text/plain is commonly used by web servers * for files whose correct type is unknown, we don't @@ -1855,7 +2015,7 @@ static jb_err server_content_type(struct client_state *csp, char **header) */ if ((strstr(*header, "text/") && !strstr(*header, "plain")) || strstr(*header, "xml") - || strstr(*header, "application/x-javascript")) + || strstr(*header, "script")) { csp->content_type |= CT_TEXT; } @@ -1871,8 +2031,8 @@ static jb_err server_content_type(struct client_state *csp, char **header) if (csp->action->flags & ACTION_CONTENT_TYPE_OVERWRITE) { /* - * Make sure the user doesn't accidently - * change the content type of binary documents. + * Make sure the user doesn't accidentally + * change the content type of binary documents. */ if ((csp->content_type & CT_TEXT) || (csp->action->flags & ACTION_FORCE_TEXT_MODE)) { @@ -1965,16 +2125,16 @@ static jb_err server_transfer_coding(struct client_state *csp, char **header) * * Function : server_content_encoding * - * Description : This function is run twice for each request, - * unless FEATURE_ZLIB and filtering are disabled. + * Description : Used to check if the content is compressed, and if + * FEATURE_ZLIB is disabled, filtering is disabled as + * well. * - * The first run is used to check if the content - * is compressed, if FEATURE_ZLIB is disabled - * filtering is then disabled as well, if FEATURE_ZLIB - * is enabled the content is marked for decompression. - * - * The second run is used to remove the Content-Encoding - * header if the decompression was successful. + * If FEATURE_ZLIB is enabled and the compression type + * supported, the content is marked for decompression. + * + * XXX: Doesn't properly deal with multiple or with + * unsupported but unknown encodings. + * Is case-sensitive but shouldn't be. * * Parameters : * 1 : csp = Current client state (buffers, headers, etc...) @@ -1990,19 +2150,25 @@ static jb_err server_transfer_coding(struct client_state *csp, char **header) static jb_err server_content_encoding(struct client_state *csp, char **header) { #ifdef FEATURE_ZLIB - if ((csp->flags & CSP_FLAG_MODIFIED) - && (csp->content_type & (CT_GZIP | CT_DEFLATE))) + if (strstr(*header, "sdch")) { /* - * We successfully decompressed the content, - * and have to clean the header now, so the - * client no longer expects compressed data.. - * - * XXX: There is a difference between cleaning - * and removing it completely. + * Shared Dictionary Compression over HTTP isn't supported, + * filtering it anyway is pretty much guaranteed to mess up + * the encoding. */ - log_error(LOG_LEVEL_HEADER, "Crunching: %s", *header); - freez(*header); + csp->content_type |= CT_TABOO; + + /* + * Log a warning if the user expects the content to be filtered. + */ + if ((csp->rlist != NULL) && + (!list_is_empty(csp->action->multi[ACTION_MULTI_FILTER]))) + { + log_error(LOG_LEVEL_INFO, + "SDCH-compressed content detected, content filtering disabled. " + "Consider suppressing SDCH offers made by the client."); + } } else if (strstr(*header, "gzip")) { @@ -2023,7 +2189,16 @@ static jb_err server_content_encoding(struct client_state *csp, char **header) csp->content_type |= CT_TABOO; } #else /* !defined(FEATURE_ZLIB) */ - if (strstr(*header, "gzip") || strstr(*header, "compress") || strstr(*header, "deflate")) + /* + * XXX: Using a black list here isn't the right approach. + * + * In case of SDCH, building with zlib support isn't + * going to help. + */ + if (strstr(*header, "gzip") || + strstr(*header, "compress") || + strstr(*header, "deflate") || + strstr(*header, "sdch")) { /* * Body is compressed, turn off pcrs and gif filtering. @@ -2049,6 +2224,49 @@ static jb_err server_content_encoding(struct client_state *csp, char **header) } +#ifdef FEATURE_ZLIB +/********************************************************************* + * + * Function : server_adjust_content_encoding + * + * Description : Remove the Content-Encoding header if the + * decompression was successful and the content + * has been modifed. + * + * Parameters : + * 1 : csp = Current client state (buffers, headers, etc...) + * 2 : header = On input, pointer to header to modify. + * On output, pointer to the modified header, or NULL + * to remove the header. This function frees the + * original string if necessary. + * + * Returns : JB_ERR_OK on success, or + * JB_ERR_MEMORY on out-of-memory error. + * + *********************************************************************/ +static jb_err server_adjust_content_encoding(struct client_state *csp, char **header) +{ + if ((csp->flags & CSP_FLAG_MODIFIED) + && (csp->content_type & (CT_GZIP | CT_DEFLATE))) + { + /* + * We successfully decompressed the content, + * and have to clean the header now, so the + * client no longer expects compressed data. + * + * XXX: There is a difference between cleaning + * and removing it completely. + */ + log_error(LOG_LEVEL_HEADER, "Crunching: %s", *header); + freez(*header); + } + + return JB_ERR_OK; + +} +#endif /* defined(FEATURE_ZLIB) */ + + /********************************************************************* * * Function : server_adjust_content_length @@ -2112,7 +2330,7 @@ static jb_err server_save_content_length(struct client_state *csp, char **header assert(*(*header+14) == ':'); - if (1 != sscanf(*header+14, ": %llu", &content_length)) + if (JB_ERR_OK != get_content_length(*header, &content_length)) { log_error(LOG_LEVEL_ERROR, "Crunching invalid header: %s", *header); freez(*header); @@ -2202,7 +2420,7 @@ static jb_err server_content_disposition(struct client_state *csp, char **header return JB_ERR_OK; } else - { + { /* * Replacing Content-Disposition header */ @@ -2242,16 +2460,9 @@ static jb_err server_content_disposition(struct client_state *csp, char **header static jb_err server_last_modified(struct client_state *csp, char **header) { const char *newval; - char buf[BUFFER_SIZE]; - + time_t last_modified; char newheader[50]; -#ifdef HAVE_GMTIME_R - struct tm gmt; -#endif - struct tm *timeptr = NULL; - time_t now, last_modified; - long int days, hours, minutes, seconds; - + /* * Are we messing with the Last-Modified header? */ @@ -2273,18 +2484,19 @@ static jb_err server_last_modified(struct client_state *csp, char **header) return JB_ERR_OK; } else if (0 == strcmpic(newval, "reset-to-request-time")) - { + { /* * Setting Last-Modified Header to now. */ + char buf[30]; get_http_time(0, buf, sizeof(buf)); freez(*header); *header = strdup("Last-Modified: "); - string_append(header, buf); + string_append(header, buf); if (*header == NULL) { - log_error(LOG_LEVEL_HEADER, "Insufficient memory. Last-Modified header got lost, boohoo."); + log_error(LOG_LEVEL_HEADER, "Insufficient memory. Last-Modified header got lost, boohoo."); } else { @@ -2296,16 +2508,7 @@ static jb_err server_last_modified(struct client_state *csp, char **header) const char *header_time = *header + sizeof("Last-Modified:"); log_error(LOG_LEVEL_HEADER, "Randomizing: %s", *header); - now = time(NULL); -#ifdef HAVE_GMTIME_R - gmtime_r(&now, &gmt); -#elif defined(MUTEX_LOCKS_AVAILABLE) - privoxy_mutex_lock(&gmtime_mutex); - gmtime(&now); - privoxy_mutex_unlock(&gmtime_mutex); -#else - gmtime(&now); -#endif + if (JB_ERR_OK != parse_header_time(header_time, &last_modified)) { log_error(LOG_LEVEL_HEADER, "Couldn't parse: %s in %s (crunching!)", header_time, *header); @@ -2313,14 +2516,22 @@ static jb_err server_last_modified(struct client_state *csp, char **header) } else { - long int rtime = (long int)difftime(now, last_modified); + time_t now; + struct tm *timeptr = NULL; + long int rtime; +#ifdef HAVE_GMTIME_R + struct tm gmt; +#endif + now = time(NULL); + rtime = (long int)difftime(now, last_modified); if (rtime) { + long int days, hours, minutes, seconds; const int negative_delta = (rtime < 0); if (negative_delta) { - rtime *= -1; + rtime *= -1; log_error(LOG_LEVEL_HEADER, "Server time in the future."); } rtime = pick_from_range(rtime); @@ -2355,7 +2566,7 @@ static jb_err server_last_modified(struct client_state *csp, char **header) if (*header == NULL) { log_error(LOG_LEVEL_ERROR, "Insufficient memory, header crunched without replacement."); - return JB_ERR_MEMORY; + return JB_ERR_MEMORY; } days = rtime / (3600 * 24); @@ -2400,25 +2611,17 @@ static jb_err server_last_modified(struct client_state *csp, char **header) *********************************************************************/ static jb_err client_accept_encoding(struct client_state *csp, char **header) { +#ifdef FEATURE_COMPRESSION + if ((csp->config->feature_flags & RUNTIME_FEATURE_COMPRESSION) + && strstr(*header, "deflate")) + { + csp->flags |= CSP_FLAG_CLIENT_SUPPORTS_DEFLATE; + } +#endif if ((csp->action->flags & ACTION_NO_COMPRESSION) != 0) { log_error(LOG_LEVEL_HEADER, "Suppressed offer to compress content"); - freez(*header); - - /* Temporarily disable the correct behaviour to - * work around a PHP bug. - * - * if (!strcmpic(csp->http->ver, "HTTP/1.1")) - * { - * *header = strdup("Accept-Encoding: identity;q=1.0, *;q=0"); - * if (*header == NULL) - * { - * return JB_ERR_MEMORY; - * } - * } - * - */ } return JB_ERR_OK; @@ -2479,7 +2682,7 @@ static jb_err client_referrer(struct client_state *csp, char **header) /* booleans for parameters we have to check multiple times */ int parameter_conditional_block; int parameter_conditional_forge; - + #ifdef FEATURE_FORCE_LOAD /* * Since the referrer can include the prefix even @@ -2577,18 +2780,18 @@ static jb_err client_accept_language(struct client_state *csp, char **header) return JB_ERR_OK; } else - { + { /* * Replacing Accept-Language header */ freez(*header); *header = strdup("Accept-Language: "); - string_append(header, newval); + string_append(header, newval); if (*header == NULL) { log_error(LOG_LEVEL_ERROR, - "Insufficient memory. Accept-Language header crunched without replacement."); + "Insufficient memory. Accept-Language header crunched without replacement."); } else { @@ -2629,7 +2832,7 @@ static jb_err crunch_client_header(struct client_state *csp, char **header) /* Is the current header the lucky one? */ if (strstr(*header, crunch_pattern)) { - log_error(LOG_LEVEL_HEADER, "Crunching client header: %s (contains: %s)", *header, crunch_pattern); + log_error(LOG_LEVEL_HEADER, "Crunching client header: %s (contains: %s)", *header, crunch_pattern); freez(*header); } } @@ -2934,10 +3137,10 @@ static jb_err client_host(struct client_state *csp, char **header) return JB_ERR_OK; } - if (!csp->http->hostport || (*csp->http->hostport == '*') || + if (!csp->http->hostport || (*csp->http->hostport == '*') || *csp->http->hostport == ' ' || *csp->http->hostport == '\0') { - + if (NULL == (p = strdup((*header)+6))) { return JB_ERR_MEMORY; @@ -3000,14 +3203,13 @@ static jb_err client_if_modified_since(struct client_state *csp, char **header) struct tm gmt; #endif struct tm *timeptr = NULL; - time_t tm = 0; + time_t tm = 0; const char *newval; - long int hours, minutes, seconds; char * endptr; - + if ( 0 == strcmpic(*header, "If-Modified-Since: Wed, 08 Jun 1955 12:00:00 GMT")) { - /* + /* * The client got an error message because of a temporary problem, * the problem is gone and the client now tries to revalidate our * error message on the real server. The revalidation would always @@ -3037,6 +3239,7 @@ static jb_err client_if_modified_since(struct client_state *csp, char **header) } else { + long int hours, minutes, seconds; long int rtime = strtol(newval, &endptr, 0); const int negative_range = (rtime < 0); @@ -3046,7 +3249,7 @@ static jb_err client_if_modified_since(struct client_state *csp, char **header) *header, rtime, (rtime == 1 || rtime == -1) ? "e": "es"); if (negative_range) { - rtime *= -1; + rtime *= -1; } rtime *= 60; rtime = pick_from_range(rtime); @@ -3083,7 +3286,7 @@ static jb_err client_if_modified_since(struct client_state *csp, char **header) if (*header == NULL) { log_error(LOG_LEVEL_HEADER, "Insufficient memory, header crunched without replacement."); - return JB_ERR_MEMORY; + return JB_ERR_MEMORY; } hours = rtime / 3600; @@ -3123,7 +3326,7 @@ static jb_err client_if_modified_since(struct client_state *csp, char **header) static jb_err client_if_none_match(struct client_state *csp, char **header) { if (csp->action->flags & ACTION_CRUNCH_IF_NONE_MATCH) - { + { log_error(LOG_LEVEL_HEADER, "Crunching %s", *header); freez(*header); } @@ -3165,7 +3368,7 @@ jb_err client_x_filter(struct client_state *csp, char **header) "force-text-mode overruled the client's request to fetch without filtering!"); } else - { + { csp->content_type = CT_TABOO; /* XXX: This hack shouldn't be necessary */ csp->flags |= CSP_FLAG_NO_FILTERING; log_error(LOG_LEVEL_HEADER, "Accepted the client's request to fetch without filtering."); @@ -3174,7 +3377,7 @@ jb_err client_x_filter(struct client_state *csp, char **header) freez(*header); } } - return JB_ERR_OK; + return JB_ERR_OK; } @@ -3208,7 +3411,7 @@ static jb_err client_range(struct client_state *csp, char **header) freez(*header); } - return JB_ERR_OK; + return JB_ERR_OK; } /* the following functions add headers directly to the header list */ @@ -3265,36 +3468,6 @@ static jb_err client_host_adder(struct client_state *csp) } -#if 0 -/********************************************************************* - * - * Function : client_accept_encoding_adder - * - * Description : Add an Accept-Encoding header to the client's request - * that disables compression if the action applies, and - * the header is not already there. Called from `sed'. - * Note: For HTTP/1.0, the absence of the header is enough. - * - * Parameters : - * 1 : csp = Current client state (buffers, headers, etc...) - * - * Returns : JB_ERR_OK on success, or - * JB_ERR_MEMORY on out-of-memory error. - * - *********************************************************************/ -static jb_err client_accept_encoding_adder(struct client_state *csp) -{ - if ( ((csp->action->flags & ACTION_NO_COMPRESSION) != 0) - && (!strcmpic(csp->http->ver, "HTTP/1.1")) ) - { - return enlist_unique(csp->headers, "Accept-Encoding: identity;q=1.0, *;q=0", 16); - } - - return JB_ERR_OK; -} -#endif - - /********************************************************************* * * Function : client_xtra_adder @@ -3379,7 +3552,7 @@ static jb_err client_x_forwarded_for_adder(struct client_state *csp) * * Function : server_connection_adder * - * Description : Adds an appropiate "Connection:" header to csp->headers + * Description : Adds an appropriate "Connection:" header to csp->headers * unless the header was already present. Called from `sed'. * * Parameters : @@ -3393,7 +3566,7 @@ static jb_err server_connection_adder(struct client_state *csp) { const unsigned int flags = csp->flags; const char *response_status_line = csp->headers->first->str; - const char *wanted_header = get_appropiate_connection_header(csp); + static const char connection_close[] = "Connection: close"; if ((flags & CSP_FLAG_CLIENT_HEADER_PARSING_DONE) && (flags & CSP_FLAG_SERVER_CONNECTION_HEADER_SET)) @@ -3416,11 +3589,12 @@ static jb_err server_connection_adder(struct client_state *csp) log_error(LOG_LEVEL_HEADER, "A HTTP/1.1 response " "without Connection header implies keep-alive."); csp->flags |= CSP_FLAG_SERVER_CONNECTION_KEEP_ALIVE; + return JB_ERR_OK; } - log_error(LOG_LEVEL_HEADER, "Adding: %s", wanted_header); + log_error(LOG_LEVEL_HEADER, "Adding: %s", connection_close); - return enlist(csp->headers, wanted_header); + return enlist(csp->headers, connection_close); } @@ -3431,7 +3605,7 @@ static jb_err server_connection_adder(struct client_state *csp) * * Description : Adds a "Proxy-Connection: keep-alive" header to * csp->headers if the client asked for keep-alive. - * XXX: We should reuse existant ones. + * XXX: We should reuse existent ones. * * Parameters : * 1 : csp = Current client state (buffers, headers, etc...) @@ -3446,7 +3620,8 @@ static jb_err server_proxy_connection_adder(struct client_state *csp) jb_err err = JB_ERR_OK; if ((csp->flags & CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE) - && !(csp->flags & CSP_FLAG_SERVER_SOCKET_TAINTED)) + && !(csp->flags & CSP_FLAG_SERVER_SOCKET_TAINTED) + && !(csp->flags & CSP_FLAG_SERVER_PROXY_CONNECTION_HEADER_SET)) { log_error(LOG_LEVEL_HEADER, "Adding: %s", proxy_connection_header); err = enlist(csp->headers, proxy_connection_header); @@ -3473,7 +3648,7 @@ static jb_err server_proxy_connection_adder(struct client_state *csp) *********************************************************************/ static jb_err client_connection_header_adder(struct client_state *csp) { - const char *wanted_header = get_appropiate_connection_header(csp); + static const char connection_close[] = "Connection: close"; if (!(csp->flags & CSP_FLAG_CLIENT_HEADER_PARSING_DONE) && (csp->flags & CSP_FLAG_CLIENT_CONNECTION_HEADER_SET)) @@ -3487,12 +3662,13 @@ static jb_err client_connection_header_adder(struct client_state *csp) && !strcmpic(csp->http->ver, "HTTP/1.1")) { csp->flags |= CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE; + return JB_ERR_OK; } #endif /* FEATURE_CONNECTION_KEEP_ALIVE */ - log_error(LOG_LEVEL_HEADER, "Adding: %s", wanted_header); + log_error(LOG_LEVEL_HEADER, "Adding: %s", connection_close); - return enlist(csp->headers, wanted_header); + return enlist(csp->headers, connection_close); } @@ -3553,8 +3729,7 @@ static jb_err server_http(struct client_state *csp, char **header) * Function : server_set_cookie * * Description : Handle the server "cookie" header properly. - * Log cookie to the jar file. Then "crunch", - * accept or rewrite it to a session cookie. + * Crunch, accept or rewrite it to a session cookie. * Called from `sed'. * * TODO: Allow the user to specify a new expiration @@ -3575,7 +3750,7 @@ static jb_err server_http(struct client_state *csp, char **header) static jb_err server_set_cookie(struct client_state *csp, char **header) { time_t now; - time_t cookie_time; + time_t cookie_time; time(&now); @@ -3632,6 +3807,17 @@ static jb_err server_set_cookie(struct client_state *csp, char **header) { char *expiration_date = cur_tag + 8; /* Skip "[Ee]xpires=" */ + if ((expiration_date[0] == '"') + && (expiration_date[1] != '\0')) + { + /* + * Skip quotation mark. RFC 2109 10.1.2 seems to hint + * that the expiration date isn't supposed to be quoted, + * but some servers do it anyway. + */ + expiration_date++; + } + /* Did we detect the date properly? */ if (JB_ERR_OK != parse_header_time(expiration_date, &cookie_time)) { @@ -3731,7 +3917,7 @@ static jb_err server_set_cookie(struct client_state *csp, char **header) * * Function : strclean * - * Description : In-Situ-Eliminate all occurances of substring in + * Description : In-Situ-Eliminate all occurrences of substring in * string * * Parameters : @@ -3774,7 +3960,7 @@ int strclean(char *string, const char *substring) * to get the numerical respresentation. * * Parameters : - * 1 : header_time = HTTP header time as string. + * 1 : header_time = HTTP header time as string. * 2 : result = storage for header_time in seconds * * Returns : JB_ERR_OK if the time format was recognized, or @@ -3784,34 +3970,49 @@ int strclean(char *string, const char *substring) static jb_err parse_header_time(const char *header_time, time_t *result) { struct tm gmt; - /* - * Zero out gmt to prevent time zone offsets. - * - * While this is only necessary on some platforms - * (mingw32 for example), I don't know how to - * detect these automatically and doing it everywhere - * shouldn't hurt. + * Checking for two-digit years first in an + * attempt to work around GNU libc's strptime() + * reporting negative year values when using %Y. */ - memset(&gmt, 0, sizeof(gmt)); - - /* Tue, 02 Jun 2037 20:00:00 */ - if ((NULL == strptime(header_time, "%a, %d %b %Y %H:%M:%S", &gmt)) - /* Tue, 02-Jun-2037 20:00:00 */ - && (NULL == strptime(header_time, "%a, %d-%b-%Y %H:%M:%S", &gmt)) - /* Tue, 02-Jun-37 20:00:00 */ - && (NULL == strptime(header_time, "%a, %d-%b-%y %H:%M:%S", &gmt)) - /* Tuesday, 02-Jun-2037 20:00:00 */ - && (NULL == strptime(header_time, "%A, %d-%b-%Y %H:%M:%S", &gmt)) - /* Tuesday Jun 02 20:00:00 2037 */ - && (NULL == strptime(header_time, "%A %b %d %H:%M:%S %Y", &gmt))) + static const char * const time_formats[] = { + /* Tue, 02-Jun-37 20:00:00 */ + "%a, %d-%b-%y %H:%M:%S", + /* Tue, 02 Jun 2037 20:00:00 */ + "%a, %d %b %Y %H:%M:%S", + /* Tue, 02-Jun-2037 20:00:00 */ + "%a, %d-%b-%Y %H:%M:%S", + /* Tuesday, 02-Jun-2037 20:00:00 */ + "%A, %d-%b-%Y %H:%M:%S", + /* Tuesday Jun 02 20:00:00 2037 */ + "%A %b %d %H:%M:%S %Y" + }; + unsigned int i; + + for (i = 0; i < SZ(time_formats); i++) { - return JB_ERR_PARSE; - } + /* + * Zero out gmt to prevent time zone offsets. + * Documented to be required for GNU libc. + */ + memset(&gmt, 0, sizeof(gmt)); - *result = timegm(&gmt); + if (NULL != strptime(header_time, time_formats[i], &gmt)) + { + /* Sanity check for GNU libc. */ + if (gmt.tm_year < 0) + { + log_error(LOG_LEVEL_HEADER, + "Failed to parse '%s' using '%s'. Moving on.", + header_time, time_formats[i]); + continue; + } + *result = timegm(&gmt); + return JB_ERR_OK; + } + } - return JB_ERR_OK; + return JB_ERR_PARSE; } @@ -3830,7 +4031,7 @@ static jb_err parse_header_time(const char *header_time, time_t *result) * Parameters : * 1 : headers = List of headers (one of them hopefully being * the "Host:" header) - * 2 : http = storage for the result (host, port and hostport). + * 2 : http = storage for the result (host, port and hostport). * * Returns : JB_ERR_MEMORY in case of memory problems, * JB_ERR_PARSE if the host header couldn't be found, @@ -3851,7 +4052,8 @@ jb_err get_destination_from_headers(const struct list *headers, struct http_requ return JB_ERR_PARSE; } - if (NULL == (p = strdup((host)))) + p = strdup(host); + if (NULL == p) { log_error(LOG_LEVEL_ERROR, "Out of memory while parsing \"Host:\" header"); return JB_ERR_MEMORY; @@ -3904,7 +4106,7 @@ jb_err get_destination_from_headers(const struct list *headers, struct http_requ * * Description : Helper for client_referrer to forge a referer as * 'http://[hostname:port/' to fool stupid - * checks for in-site links + * checks for in-site links * * Parameters : * 1 : header = Pointer to header pointer @@ -3995,7 +4197,7 @@ static jb_err handle_conditional_hide_referrer_parameter(char **header, const char *host, const int parameter_conditional_block) { char *referer = strdup(*header); - const size_t hostlenght = strlen(host); + const size_t hostlength = strlen(host); const char *referer_url = NULL; if (NULL == referer) @@ -4005,14 +4207,14 @@ static jb_err handle_conditional_hide_referrer_parameter(char **header, } /* referer begins with 'Referer: http[s]://' */ - if ((hostlenght+17) < strlen(referer)) + if ((hostlength+17) < strlen(referer)) { /* * Shorten referer to make sure the referer is blocked * if www.example.org/www.example.com-shall-see-the-referer/ * links to www.example.com/ */ - referer[hostlenght+17] = '\0'; + referer[hostlength+17] = '\0'; } referer_url = strstr(referer, "http://"); if ((NULL == referer_url) || (NULL == strstr(referer_url, host))) @@ -4037,37 +4239,6 @@ static jb_err handle_conditional_hide_referrer_parameter(char **header, } -/********************************************************************* - * - * Function : get_appropiate_connection_header - * - * Description : Returns an appropiate Connection header - * depending on whether or not we try to keep - * the connection to the server alive. - * - * Parameters : - * 1 : csp = Current client state (buffers, headers, etc...) - * - * Returns : Pointer to statically allocated header buffer. - * - *********************************************************************/ -static const char *get_appropiate_connection_header(const struct client_state *csp) -{ - static const char connection_keep_alive[] = "Connection: keep-alive"; - static const char connection_close[] = "Connection: close"; - - if ((csp->config->feature_flags & RUNTIME_FEATURE_CONNECTION_KEEP_ALIVE) -#ifdef FEATURE_CONNECTION_KEEP_ALIVE - && !(csp->flags & CSP_FLAG_SERVER_SOCKET_TAINTED) -#endif - && (csp->http->ssl == 0)) - { - return connection_keep_alive; - } - return connection_close; -} - - /********************************************************************* * * Function : create_content_length_header