From: jongfoster Date: Sun, 16 Sep 2001 15:02:35 +0000 (+0000) Subject: Adding i.j.b/robots.txt. X-Git-Tag: v_2_9_9~78 X-Git-Url: http://www.privoxy.org/gitweb/?p=privoxy.git;a=commitdiff_plain;h=d8b38820d095bacee7530284ac580bf644a783e5 Adding i.j.b/robots.txt. Inlining add_stats() since it's only ever called from one place. --- diff --git a/cgi.c b/cgi.c index 6860dd91..ea63193e 100644 --- a/cgi.c +++ b/cgi.c @@ -1,4 +1,4 @@ -const char cgi_rcs[] = "$Id: cgi.c,v 1.23 2001/09/16 11:16:05 jongfoster Exp $"; +const char cgi_rcs[] = "$Id: cgi.c,v 1.24 2001/09/16 11:38:01 jongfoster Exp $"; /********************************************************************* * * File : $Source: /cvsroot/ijbswa/current/cgi.c,v $ @@ -36,6 +36,14 @@ const char cgi_rcs[] = "$Id: cgi.c,v 1.23 2001/09/16 11:16:05 jongfoster Exp $"; * * Revisions : * $Log: cgi.c,v $ + * Revision 1.24 2001/09/16 11:38:01 jongfoster + * Splitting fill_template() into 2 functions: + * template_load() loads the file + * template_fill() performs the PCRS regexps. + * This is because the CGI edit interface has a "table row" + * template which is used many times in the page - this + * change means it's only loaded from disk once. + * * Revision 1.23 2001/09/16 11:16:05 jongfoster * Better error handling in dispatch_cgi() and parse_cgi_parameters() * @@ -196,6 +204,9 @@ const char cgi_rcs[] = "$Id: cgi.c,v 1.23 2001/09/16 11:16:05 jongfoster Exp $"; const char cgi_h_rcs[] = CGI_H_VERSION; const struct cgi_dispatcher cgi_dispatcher[] = { + { "robots.txt", + 10, cgi_robots_txt, + "HIDE Sends a robots.txt file to tell robots to go away." }, { "show-status", 11, cgi_show_status, "Show information about the current configuration" }, @@ -541,6 +552,11 @@ int cgi_show_status(struct client_state *csp, struct http_response *rsp, char * p; const char * filename = NULL; char * file_description = NULL; +#ifdef FEATURE_STATISTICS + float perc_rej; /* Percentage of http requests rejected */ + int local_urls_read; + int local_urls_rejected; +#endif /* ndef FEATURE_STATISTICS */ struct map * exports = default_exports(csp, "show-status"); @@ -618,7 +634,40 @@ int cgi_show_status(struct client_state *csp, struct http_response *rsp, show_defines(exports); #ifdef FEATURE_STATISTICS - add_stats(exports); + local_urls_read = urls_read; + local_urls_rejected = urls_rejected; + + /* + * Need to alter the stats not to include the fetch of this + * page. + * + * Can't do following thread safely! doh! + * + * urls_read--; + * urls_rejected--; * This will be incremented subsequently * + */ + + if (local_urls_read == 0) + { + map_block_killer(exports, "have-stats"); + } + else + { + map_block_killer(exports, "have-no-stats"); + + perc_rej = (float)local_urls_rejected * 100.0F / + (float)local_urls_read; + + sprintf(buf, "%d", local_urls_read); + map(exports, "requests-received", 1, buf, 1); + + sprintf(buf, "%d", local_urls_rejected); + map(exports, "requests-blocked", 1, buf, 1); + + sprintf(buf, "%6.2f", perc_rej); + map(exports, "percent-blocked", 1, buf, 1); + } + #else /* ndef FEATURE_STATISTICS */ map_block_killer(exports, "statistics"); #endif /* ndef FEATURE_STATISTICS */ @@ -1412,61 +1461,45 @@ char *dump_map(const struct map *the_map) } -#ifdef FEATURE_STATISTICS /********************************************************************* * - * Function : add_stats + * Function : cgi_robots_txt * - * Description : Add the blocking statistics to a given map. + * Description : CGI function to return "/robots.txt". * * Parameters : - * 1 : exports = map to write to. + * 1 : csp = Current client state (buffers, headers, etc...) + * 2 : rsp = http_response data structure for output + * 3 : parameters = map of cgi parameters * - * Returns : pointer to extended map + * CGI Parameters : None + * + * Returns : 0 * *********************************************************************/ -struct map *add_stats(struct map *exports) +int cgi_robots_txt(struct client_state *csp, struct http_response *rsp, + struct map *parameters) { - float perc_rej; /* Percentage of http requests rejected */ - char buf[1000]; - int local_urls_read = urls_read; - int local_urls_rejected = urls_rejected; - - /* - * Need to alter the stats not to include the fetch of this - * page. - * - * Can't do following thread safely! doh! - * - * urls_read--; - * urls_rejected--; * This will be incremented subsequently * - */ - - if (local_urls_read == 0) - { - map_block_killer(exports, "have-stats"); - } - else - { - map_block_killer(exports, "have-no-stats"); - - perc_rej = (float)local_urls_rejected * 100.0F / - (float)local_urls_read; + char buf[100]; - sprintf(buf, "%d", local_urls_read); - map(exports, "requests-received", 1, buf, 1); + rsp->body = strdup( + "# This is the Internet Junkbuster control interface.\n" + "# It isn't very useful to index it, and you're likely to break stuff.\n" + "# So go away!\n" + "\n" + "User-agent: *\n" + "Disallow: /\n" + "\n"); - sprintf(buf, "%d", local_urls_rejected); - map(exports, "requests-blocked", 1, buf, 1); + enlist_unique(rsp->headers, "Content-Type: text/plain", 13); - sprintf(buf, "%6.2f", perc_rej); - map(exports, "percent-blocked", 1, buf, 1); - } + rsp->is_static = 1; - return(exports); + get_http_time(7 * 24 * 60 * 60, buf); /* 7 days into future */ + enlist_unique_header(rsp->headers, "Expires", buf); + return 0; } -#endif /* def FEATURE_STATISTICS */ /* diff --git a/cgi.h b/cgi.h index 5cc92d76..4f155d64 100644 --- a/cgi.h +++ b/cgi.h @@ -1,6 +1,6 @@ #ifndef CGI_H_INCLUDED #define CGI_H_INCLUDED -#define CGI_H_VERSION "$Id: cgi.h,v 1.13 2001/09/16 11:00:10 jongfoster Exp $" +#define CGI_H_VERSION "$Id: cgi.h,v 1.14 2001/09/16 11:38:02 jongfoster Exp $" /********************************************************************* * * File : $Source: /cvsroot/ijbswa/current/cgi.h,v $ @@ -38,6 +38,14 @@ * * Revisions : * $Log: cgi.h,v $ + * Revision 1.14 2001/09/16 11:38:02 jongfoster + * Splitting fill_template() into 2 functions: + * template_load() loads the file + * template_fill() performs the PCRS regexps. + * This is because the CGI edit interface has a "table row" + * template which is used many times in the page - this + * change means it's only loaded from disk once. + * * Revision 1.13 2001/09/16 11:00:10 jongfoster * New function alloc_http_response, for symmetry with free_http_response * @@ -105,19 +113,29 @@ extern struct map *parse_cgi_parameters(char *argstring); /* * CGI functions */ -extern int cgi_show_version(struct client_state *csp, struct http_response *rsp, - struct map *parameters); -extern int cgi_default(struct client_state *csp, struct http_response *rsp, - struct map *parameters); -extern int cgi_show_status(struct client_state *csp, struct http_response *rsp, - struct map *parameters); -extern int cgi_show_url_info(struct client_state *csp, struct http_response *rsp, - struct map *parameters); -extern int cgi_send_banner(struct client_state *csp, struct http_response *rsp, - struct map *parameters); +extern int cgi_default (struct client_state *csp, + struct http_response *rsp, + struct map *parameters); +extern int cgi_robots_txt (struct client_state *csp, + struct http_response *rsp, + struct map *parameters); +extern int cgi_send_banner (struct client_state *csp, + struct http_response *rsp, + struct map *parameters); +extern int cgi_show_status (struct client_state *csp, + struct http_response *rsp, + struct map *parameters); +extern int cgi_show_url_info (struct client_state *csp, + struct http_response *rsp, + struct map *parameters); +extern int cgi_show_version (struct client_state *csp, + struct http_response *rsp, + struct map *parameters); /* Not exactly a CGI */ -extern struct http_response *error_response(struct client_state *csp, const char *template, int err); +extern struct http_response * error_response(struct client_state *csp, + const char *templatename, + int err); /* * CGI support functions @@ -141,10 +159,6 @@ extern void template_fill(char ** template_ptr, struct map *exports); extern char *make_menu(const char *self); extern char *dump_map(const struct map *map); -#ifdef FEATURE_STATISTICS -extern struct map *add_stats(struct map *exports); -#endif /* def FEATURE_STATISTICS */ - /* * Some images. */