X-Git-Url: http://www.privoxy.org/gitweb/?p=privoxy.git;a=blobdiff_plain;f=filters.c;h=91c985490b4fc26b7b827c2f678ce7b71ef887b3;hp=0d36485d8fc4c57d044ecbc53cb5d9f14a7818a1;hb=5d67369224f780b21d94cacf31dbcdf288c7ed6c;hpb=767afb7f5d7062d75385b9ab3afdad47d0249a39 diff --git a/filters.c b/filters.c index 0d36485d..91c98549 100644 --- a/filters.c +++ b/filters.c @@ -4,7 +4,7 @@ * * Purpose : Declares functions to parse/crunch headers and pages. * - * Copyright : Written by and Copyright (C) 2001-2016 the + * Copyright : Written by and Copyright (C) 2001-2020 the * Privoxy team. https://www.privoxy.org/ * * Based on the Internet Junkbuster originally written @@ -42,18 +42,12 @@ #include #ifndef _WIN32 -#ifndef __OS2__ #include -#endif /* ndef __OS2__ */ #include #else #include #endif /* ndef _WIN32 */ -#ifdef __OS2__ -#include -#endif /* def __OS2__ */ - #include "project.h" #include "filters.h" #include "encode.h" @@ -72,6 +66,9 @@ #ifdef FEATURE_CLIENT_TAGS #include "client-tags.h" #endif +#ifdef FEATURE_HTTPS_INSPECTION +#include "ssl.h" +#endif #ifdef _WIN32 #include "win32.h" @@ -87,6 +84,10 @@ static void apply_url_actions(struct current_action_spec *action, #endif struct url_actions *b); +#ifdef FEATURE_EXTENDED_STATISTICS +static void increment_block_reason_counter(const char *block_reason); +#endif + #ifdef FEATURE_ACL #ifdef HAVE_RFC2553 /********************************************************************* @@ -206,7 +207,7 @@ static int match_sockaddr(const struct sockaddr_storage *network, return 0; } - /* TODO: Optimize by checking by words insted of octets */ + /* TODO: Optimize by checking by words instead of octets */ for (i = 0; (i < addr_len) && netmask_addr[i]; i++) { if ((network_addr[i] & netmask_addr[i]) != @@ -505,7 +506,7 @@ int acl_addr(const char *aspec, struct access_control_addr *aca) * * Description : Check to see if CONNECT requests to the destination * port of this request are forbidden. The check is - * independend of the actual request method. + * independent of the actual request method. * * Parameters : * 1 : csp = Current client state (buffers, headers, etc...) @@ -557,6 +558,13 @@ struct http_response *block_url(struct client_state *csp) return cgi_error_memory(); } +#ifdef FEATURE_EXTENDED_STATISTICS + if (csp->action->string[ACTION_STRING_BLOCK] != NULL) + { + increment_block_reason_counter(csp->action->string[ACTION_STRING_BLOCK]); + } +#endif + /* * If it's an image-url, send back an image or redirect * as specified by the relevant +image action @@ -1039,7 +1047,7 @@ char *rewrite_url(char *old_url, const char *pcrs_command) * the last URL found. * *********************************************************************/ -char *get_last_url(char *subject, const char *redirect_mode) +static char *get_last_url(char *subject, const char *redirect_mode) { char *new_url = NULL; char *tmp; @@ -1189,7 +1197,6 @@ struct http_response *redirect_url(struct client_state *csp) */ char * redirect_mode; #endif /* def FEATURE_FAST_REDIRECTS */ - char *old_url = NULL; char *new_url = NULL; char *redirection_string; @@ -1215,8 +1222,36 @@ struct http_response *redirect_url(struct client_state *csp) if (*redirection_string == 's') { - old_url = csp->http->url; - new_url = rewrite_url(old_url, redirection_string); + char *requested_url; + +#ifdef FEATURE_HTTPS_INSPECTION + if (client_use_ssl(csp)) + { + jb_err err; + + requested_url = strdup_or_die("https://"); + err = string_append(&requested_url, csp->http->hostport); + if (!err) err = string_append(&requested_url, csp->http->path); + if (err) + { + log_error(LOG_LEVEL_FATAL, + "Failed to rebuild URL 'https://%s%s'", + csp->http->hostport, csp->http->path); + } + } + else +#endif + { + requested_url = csp->http->url; + } + new_url = rewrite_url(requested_url, redirection_string); +#ifdef FEATURE_HTTPS_INSPECTION + if (requested_url != csp->http->url) + { + assert(client_use_ssl(csp)); + freez(requested_url); + } +#endif } else { @@ -1230,6 +1265,8 @@ struct http_response *redirect_url(struct client_state *csp) #ifdef FEATURE_FAST_REDIRECTS if ((csp->action->flags & ACTION_FAST_REDIRECTS)) { + char *old_url; + redirect_mode = csp->action->string[ACTION_STRING_FAST_REDIRECTS]; /* @@ -1532,25 +1569,34 @@ struct re_filterfile_spec *get_filter(const struct client_state *csp, /********************************************************************* * - * Function : pcrs_filter_response + * Function : pcrs_filter_impl * * Description : Execute all text substitutions from all applying - * +filter actions on the text buffer that's been - * accumulated in csp->iob->buf. + * (based on filter_response_body value) +filter + * or +client_body_filter actions on the given buffer. * * Parameters : * 1 : csp = Current client state (buffers, headers, etc...) + * 2 : filter_response_body = when TRUE execute +filter + * actions; execute +client_body_filter actions otherwise + * 3 : data = Target data + * 4 : data_len = Target data len * * Returns : a pointer to the (newly allocated) modified buffer. * or NULL if there were no hits or something went wrong * *********************************************************************/ -static char *pcrs_filter_response(struct client_state *csp) +static char *pcrs_filter_impl(const struct client_state *csp, int filter_response_body, + const char *data, size_t *data_len) { int hits = 0; size_t size, prev_size; + const int filters_idx = + filter_response_body ? ACTION_MULTI_FILTER : ACTION_MULTI_CLIENT_BODY_FILTER; + const enum filter_type filter_type = + filter_response_body ? FT_CONTENT_FILTER : FT_CLIENT_BODY_FILTER; - char *old = NULL; + const char *old = NULL; char *new = NULL; pcrs_job *job; @@ -1560,7 +1606,7 @@ static char *pcrs_filter_response(struct client_state *csp) /* * Sanity first */ - if (csp->iob->cur >= csp->iob->eod) + if (*data_len == 0) { return(NULL); } @@ -1572,15 +1618,15 @@ static char *pcrs_filter_response(struct client_state *csp) return(NULL); } - size = (size_t)(csp->iob->eod - csp->iob->cur); - old = csp->iob->cur; + size = *data_len; + old = data; /* - * For all applying +filter actions, look if a filter by that + * For all applying actions, look if a filter by that * name exists and if yes, execute it's pcrs_joblist on the * buffer. */ - for (filtername = csp->action->multi[ACTION_MULTI_FILTER]->first; + for (filtername = csp->action->multi[filters_idx]->first; filtername != NULL; filtername = filtername->next) { int current_hits = 0; /* Number of hits caused by this filter */ @@ -1588,7 +1634,7 @@ static char *pcrs_filter_response(struct client_state *csp) int job_hits = 0; /* How many hits the current job caused */ pcrs_job *joblist; - b = get_filter(csp, filtername->str, FT_CONTENT_FILTER); + b = get_filter(csp, filtername->str, filter_type); if (b == NULL) { continue; @@ -1619,7 +1665,7 @@ static char *pcrs_filter_response(struct client_state *csp) * input for the next one. */ current_hits += job_hits; - if (old != csp->iob->cur) + if (old != data) { freez(old); } @@ -1651,29 +1697,82 @@ static char *pcrs_filter_response(struct client_state *csp) if (b->dynamic) pcrs_free_joblist(joblist); - log_error(LOG_LEVEL_RE_FILTER, - "filtering %s%s (size %d) with \'%s\' produced %d hits (new size %d).", - csp->http->hostport, csp->http->path, prev_size, b->name, current_hits, size); - + if (filter_response_body) + { + log_error(LOG_LEVEL_RE_FILTER, + "filtering %s%s (size %lu) with \'%s\' produced %d hits (new size %lu).", + csp->http->hostport, csp->http->path, prev_size, b->name, current_hits, size); + } + else + { + log_error(LOG_LEVEL_RE_FILTER, + "filtering client %s request body (size %lu) with \'%s\' produced %d hits (new size %lu).", + csp->ip_addr_str, prev_size, b->name, current_hits, size); + } +#ifdef FEATURE_EXTENDED_STATISTICS + update_filter_statistics(b->name, current_hits); +#endif hits += current_hits; } /* * If there were no hits, destroy our copy and let - * chat() use the original in csp->iob + * chat() use the original content */ if (!hits) { + if (old != data && old != new) + { + freez(old); + } freez(new); return(NULL); } - csp->flags |= CSP_FLAG_MODIFIED; - csp->content_length = size; - clear_iob(csp->iob); - + *data_len = size; return(new); +} + + +/********************************************************************* + * + * Function : pcrs_filter_response_body + * + * Description : Execute all text substitutions from all applying + * +filter actions on the text buffer that's been + * accumulated in csp->iob->buf. + * + * Parameters : + * 1 : csp = Current client state (buffers, headers, etc...) + * + * Returns : a pointer to the (newly allocated) modified buffer. + * or NULL if there were no hits or something went wrong + * + *********************************************************************/ +static char *pcrs_filter_response_body(struct client_state *csp) +{ + size_t size = (size_t)(csp->iob->eod - csp->iob->cur); + + char *new = NULL; + + /* + * Sanity first + */ + if (csp->iob->cur >= csp->iob->eod) + { + return NULL; + } + + new = pcrs_filter_impl(csp, TRUE, csp->iob->cur, &size); + + if (new != NULL) + { + csp->flags |= CSP_FLAG_MODIFIED; + csp->content_length = size; + clear_iob(csp->iob); + } + return new; } @@ -1816,7 +1915,7 @@ static char *execute_external_filter(const struct client_state *csp, */ if ((*size != 0) && fwrite(content, *size, 1, fp) != 1) { - log_error(LOG_LEVEL_ERROR, "fwrite(..., %d, 1, ..) failed: %E", *size); + log_error(LOG_LEVEL_ERROR, "fwrite(..., %lu, 1, ..) failed: %E", *size); unlink(file_name); fclose(fp); return NULL; @@ -1892,7 +1991,7 @@ static char *execute_external_filter(const struct client_state *csp, { log_error(LOG_LEVEL_RE_FILTER, "Executing '%s' resulted in return value %d. " - "Read %d of up to %d bytes.", name, (ret >> 8), new_size, *size); + "Read %lu of up to %lu bytes.", name, (ret >> 8), new_size, *size); } unlink(file_name); @@ -1904,6 +2003,28 @@ static char *execute_external_filter(const struct client_state *csp, #endif /* def FEATURE_EXTERNAL_FILTERS */ +/********************************************************************* + * + * Function : pcrs_filter_request_body + * + * Description : Execute all text substitutions from all applying + * +client_body_filter actions on the given text buffer. + * + * Parameters : + * 1 : csp = Current client state (buffers, headers, etc...) + * 2 : data = Target data + * 3 : data_len = Target data len + * + * Returns : a pointer to the (newly allocated) modified buffer. + * or NULL if there were no hits or something went wrong + * + *********************************************************************/ +static char *pcrs_filter_request_body(const struct client_state *csp, const char *data, size_t *data_len) +{ + return pcrs_filter_impl(csp, FALSE, data, data_len); +} + + /********************************************************************* * * Function : gif_deanimate_response @@ -1952,7 +2073,8 @@ static char *gif_deanimate_response(struct client_state *csp) } else { - log_error(LOG_LEVEL_DEANIMATE, "Success! GIF shrunk from %d bytes to %d.", size, out->offset); + log_error(LOG_LEVEL_DEANIMATE, + "Success! GIF shrunk from %lu bytes to %lu.", size, out->offset); } csp->content_length = out->offset; csp->flags |= CSP_FLAG_MODIFIED; @@ -1991,7 +2113,7 @@ static filter_function_ptr get_filter_function(const struct client_state *csp) if ((csp->content_type & CT_TEXT) && (!list_is_empty(csp->action->multi[ACTION_MULTI_FILTER]))) { - filter_function = pcrs_filter_response; + filter_function = pcrs_filter_response_body; } else if ((csp->content_type & CT_GIF) && (csp->action->flags & ACTION_DEANIMATE)) @@ -2058,8 +2180,8 @@ static jb_err remove_chunked_transfer_coding(char *buffer, size_t *size) { log_error(LOG_LEVEL_ERROR, "Chunk size %u exceeds buffered data left. " - "Already digested %u of %u buffered bytes.", - chunksize, (unsigned int)newsize, (unsigned int)*size); + "Already digested %lu of %lu buffered bytes.", + chunksize, newsize, *size); return JB_ERR_PARSE; } @@ -2113,7 +2235,8 @@ static jb_err remove_chunked_transfer_coding(char *buffer, size_t *size) } /* XXX: Should get its own loglevel. */ - log_error(LOG_LEVEL_RE_FILTER, "De-chunking successful. Shrunk from %d to %d", *size, newsize); + log_error(LOG_LEVEL_RE_FILTER, + "De-chunking successful. Shrunk from %lu to %lu", *size, newsize); *size = newsize; @@ -2166,7 +2289,11 @@ static jb_err prepare_for_filtering(struct client_state *csp) * If the body has a supported transfer-encoding, * decompress it, adjusting size and iob->eod. */ - if (csp->content_type & (CT_GZIP|CT_DEFLATE)) + if ((csp->content_type & (CT_GZIP|CT_DEFLATE)) +#ifdef FEATURE_BROTLI + || (csp->content_type & CT_BROTLI) +#endif + ) { if (0 == csp->iob->eod - csp->iob->cur) { @@ -2184,11 +2311,14 @@ static jb_err prepare_for_filtering(struct client_state *csp) else { /* - * Unset CT_GZIP and CT_DEFLATE to remember not - * to modify the Content-Encoding header later. + * Unset content types to remember not to + * modify the Content-Encoding header later. */ csp->content_type &= ~CT_GZIP; csp->content_type &= ~CT_DEFLATE; +#ifdef FEATURE_BROTLI + csp->content_type &= ~CT_BROTLI; +#endif } } #endif @@ -2281,6 +2411,46 @@ char *execute_content_filters(struct client_state *csp) } +/********************************************************************* + * + * Function : execute_client_body_filters + * + * Description : Executes client body filters for the request that is buffered + * in the client_iob. Upon success moves client_iob cur pointer + * to the end of the processed data. + * + * Parameters : + * 1 : csp = Current client state (buffers, headers, etc...) + * 2 : content_length = content length. Upon successful filtering + * the passed value is updated with the new content length. + * + * Returns : Pointer to the modified buffer, or + * NULL if filtering failed or wasn't necessary. + * + *********************************************************************/ +char *execute_client_body_filters(struct client_state *csp, size_t *content_length) +{ + char *ret; + + assert(client_body_filters_enabled(csp->action)); + + if (content_length == 0) + { + /* + * No content, no filtering necessary. + */ + return NULL; + } + + ret = pcrs_filter_request_body(csp, csp->client_iob->cur, content_length); + if (ret != NULL) + { + csp->client_iob->cur = csp->client_iob->eod; + } + return ret; +} + + /********************************************************************* * * Function : get_url_actions @@ -2655,7 +2825,7 @@ int content_requires_filtering(struct client_state *csp) * The server didn't bother to declare a MIME-Type. * Assume it's text that can be filtered. * - * This also regulary happens with 304 responses, + * This also regularly happens with 304 responses, * therefore logging anything here would cause * too much noise. */ @@ -2704,6 +2874,25 @@ int content_filters_enabled(const struct current_action_spec *action) } +/********************************************************************* + * + * Function : client_body_filters_enabled + * + * Description : Checks whether there are any client body filters + * enabled for the current request. + * + * Parameters : + * 1 : action = Action spec to check. + * + * Returns : TRUE for yes, FALSE otherwise + * + *********************************************************************/ +int client_body_filters_enabled(const struct current_action_spec *action) +{ + return !list_is_empty(action->multi[ACTION_MULTI_CLIENT_BODY_FILTER]); +} + + /********************************************************************* * * Function : filters_available @@ -2730,6 +2919,283 @@ int filters_available(const struct client_state *csp) return FALSE; } +#ifdef FEATURE_EXTENDED_STATISTICS + +struct filter_statistics_entry +{ + char *filter; + unsigned long long executions; + unsigned long long response_bodies_modified; + unsigned long long hits; + + struct filter_statistics_entry *next; +}; + +static struct filter_statistics_entry *filter_statistics = NULL; + + +/********************************************************************* + * + * Function : register_filter_for_statistics + * + * Description : Registers a filter so we can gather statistics for + * it unless the filter has already been registered + * before. + * + * Parameters : + * 1 : filter = Name of the filter to register + * + * Returns : void + * + *********************************************************************/ +void register_filter_for_statistics(const char *filter) +{ + struct filter_statistics_entry *entry; + + privoxy_mutex_lock(&filter_statistics_mutex); + + if (filter_statistics == NULL) + { + filter_statistics = zalloc_or_die(sizeof(struct filter_statistics_entry)); + entry = filter_statistics; + entry->filter = strdup_or_die(filter); + privoxy_mutex_unlock(&filter_statistics_mutex); + return; + } + entry = filter_statistics; + while (entry != NULL) + { + if (!strcmp(entry->filter, filter)) + { + /* Already registered, nothing to do. */ + break; + } + if (entry->next == NULL) + { + entry->next = zalloc_or_die(sizeof(struct filter_statistics_entry)); + entry->next->filter = strdup_or_die(filter); + break; + } + entry = entry->next; + } + + privoxy_mutex_unlock(&filter_statistics_mutex); + +} + + +/********************************************************************* + * + * Function : update_filter_statistics + * + * Description : Updates the statistics for a filter. + * + * Parameters : + * 1 : filter = Name of the filter to update + * 2 : hits = Hit count. + * + * Returns : void + * + *********************************************************************/ +void update_filter_statistics(const char *filter, int hits) +{ + struct filter_statistics_entry *entry; + + privoxy_mutex_lock(&filter_statistics_mutex); + + entry = filter_statistics; + while (entry != NULL) + { + if (!strcmp(entry->filter, filter)) + { + entry->executions++; + if (hits != 0) + { + entry->response_bodies_modified++; + entry->hits += (unsigned)hits; + } + break; + } + entry = entry->next; + } + + privoxy_mutex_unlock(&filter_statistics_mutex); + +} + + +/********************************************************************* + * + * Function : get_filter_statistics + * + * Description : Gets the statistics for a filter. + * + * Parameters : + * 1 : filter = Name of the filter to get statistics for. + * 2 : executions = Storage for the execution count. + * 3 : response_bodies_modified = Storage for the number + * of modified response bodies. + * 4 : hits = Storage for the number of hits. + * + * Returns : void + * + *********************************************************************/ +void get_filter_statistics(const char *filter, unsigned long long *executions, + unsigned long long *response_bodies_modified, + unsigned long long *hits) +{ + struct filter_statistics_entry *entry; + + privoxy_mutex_lock(&filter_statistics_mutex); + + entry = filter_statistics; + while (entry != NULL) + { + if (!strcmp(entry->filter, filter)) + { + *executions = entry->executions; + *response_bodies_modified = entry->response_bodies_modified; + *hits = entry->hits; + break; + } + entry = entry->next; + } + + privoxy_mutex_unlock(&filter_statistics_mutex); + +} + + +struct block_statistics_entry +{ + char *block_reason; + unsigned long long count; + + struct block_statistics_entry *next; +}; + +static struct block_statistics_entry *block_statistics = NULL; + +/********************************************************************* + * + * Function : register_block_reason_for_statistics + * + * Description : Registers a block reason so we can gather statistics + * for it unless the block reason has already been + * registered before. + * + * Parameters : + * 1 : block_reason = Block reason to register + * + * Returns : void + * + *********************************************************************/ +void register_block_reason_for_statistics(const char *block_reason) +{ + struct block_statistics_entry *entry; + + privoxy_mutex_lock(&block_statistics_mutex); + + if (block_statistics == NULL) + { + block_statistics = zalloc_or_die(sizeof(struct block_statistics_entry)); + entry = block_statistics; + entry->block_reason = strdup_or_die(block_reason); + privoxy_mutex_unlock(&block_statistics_mutex); + return; + } + entry = block_statistics; + while (entry != NULL) + { + if (!strcmp(entry->block_reason, block_reason)) + { + /* Already registered, nothing to do. */ + break; + } + if (entry->next == NULL) + { + entry->next = zalloc_or_die(sizeof(struct block_statistics_entry)); + entry->next->block_reason = strdup_or_die(block_reason); + break; + } + entry = entry->next; + } + + privoxy_mutex_unlock(&block_statistics_mutex); + +} + + +/********************************************************************* + * + * Function : increment_block_reason_counter + * + * Description : Updates the counter for a block reason. + * + * Parameters : + * 1 : block_reason = Block reason to count + * + * Returns : void + * + *********************************************************************/ +static void increment_block_reason_counter(const char *block_reason) +{ + struct block_statistics_entry *entry; + + privoxy_mutex_lock(&block_statistics_mutex); + + entry = block_statistics; + while (entry != NULL) + { + if (!strcmp(entry->block_reason, block_reason)) + { + entry->count++; + break; + } + entry = entry->next; + } + + privoxy_mutex_unlock(&block_statistics_mutex); + +} + + +/********************************************************************* + * + * Function : get_block_reason_count + * + * Description : Gets number of times a block reason was used. + * + * Parameters : + * 1 : block_reason = Block reason to get statistics for. + * 2 : count = Storage for the number of times the block + * reason was used. + * + * Returns : void + * + *********************************************************************/ +void get_block_reason_count(const char *block_reason, unsigned long long *count) +{ + struct block_statistics_entry *entry; + + privoxy_mutex_lock(&block_statistics_mutex); + + entry = block_statistics; + while (entry != NULL) + { + if (!strcmp(entry->block_reason, block_reason)) + { + *count = entry->count; + break; + } + entry = entry->next; + } + + privoxy_mutex_unlock(&block_statistics_mutex); + +} + +#endif /* def FEATURE_EXTENDED_STATISTICS */ /* Local Variables: