Adding i.j.b/robots.txt.

Inlining add_stats() since it's only ever called from one place.

diff --git a/cgi.c b/cgi.c
index 6860dd9..ea63193 100644 (file)
--- a/cgi.c
+++ b/cgi.c
@@ -1,4 +1,4 @@
-const char cgi_rcs[] = "$Id: cgi.c,v 1.23 2001/09/16 11:16:05 jongfoster Exp $";
+const char cgi_rcs[] = "$Id: cgi.c,v 1.24 2001/09/16 11:38:01 jongfoster Exp $";
 /*********************************************************************
  *
  * File        :  $Source: /cvsroot/ijbswa/current/cgi.c,v $
@@ -36,6 +36,14 @@ const char cgi_rcs[] = "$Id: cgi.c,v 1.23 2001/09/16 11:16:05 jongfoster Exp $";
  *
  * Revisions   :
  *    $Log: cgi.c,v $
+ *    Revision 1.24  2001/09/16 11:38:01  jongfoster
+ *    Splitting fill_template() into 2 functions:
+ *    template_load() loads the file
+ *    template_fill() performs the PCRS regexps.
+ *    This is because the CGI edit interface has a "table row"
+ *    template which is used many times in the page - this
+ *    change means it's only loaded from disk once.
+ *
  *    Revision 1.23  2001/09/16 11:16:05  jongfoster
  *    Better error handling in dispatch_cgi() and parse_cgi_parameters()
  *
@@ -196,6 +204,9 @@ const char cgi_rcs[] = "$Id: cgi.c,v 1.23 2001/09/16 11:16:05 jongfoster Exp $";
 const char cgi_h_rcs[] = CGI_H_VERSION;
 
 const struct cgi_dispatcher cgi_dispatcher[] = {
+   { "robots.txt", 
+         10, cgi_robots_txt,  
+         "HIDE Sends a robots.txt file to tell robots to go away." }, 
    { "show-status", 
          11, cgi_show_status,  
          "Show information about the current configuration" }, 
@@ -541,6 +552,11 @@ int cgi_show_status(struct client_state *csp, struct http_response *rsp,
    char * p;
    const char * filename = NULL;
    char * file_description = NULL;
+#ifdef FEATURE_STATISTICS
+   float perc_rej;   /* Percentage of http requests rejected */
+   int local_urls_read;
+   int local_urls_rejected;
+#endif /* ndef FEATURE_STATISTICS */
 
    struct map * exports = default_exports(csp, "show-status");
 
@@ -618,7 +634,40 @@ int cgi_show_status(struct client_state *csp, struct http_response *rsp,
    show_defines(exports);
 
 #ifdef FEATURE_STATISTICS
-   add_stats(exports);
+   local_urls_read     = urls_read;
+   local_urls_rejected = urls_rejected;
+
+   /*
+    * Need to alter the stats not to include the fetch of this
+    * page.
+    *
+    * Can't do following thread safely! doh!
+    *
+    * urls_read--;
+    * urls_rejected--; * This will be incremented subsequently *
+    */
+
+   if (local_urls_read == 0)
+   {
+      map_block_killer(exports, "have-stats");
+   }
+   else
+   {
+      map_block_killer(exports, "have-no-stats");
+
+      perc_rej = (float)local_urls_rejected * 100.0F /
+            (float)local_urls_read;
+
+      sprintf(buf, "%d", local_urls_read);
+      map(exports, "requests-received", 1, buf, 1);
+
+      sprintf(buf, "%d", local_urls_rejected);
+      map(exports, "requests-blocked", 1, buf, 1);
+
+      sprintf(buf, "%6.2f", perc_rej);
+      map(exports, "percent-blocked", 1, buf, 1);
+   }
+
 #else /* ndef FEATURE_STATISTICS */
    map_block_killer(exports, "statistics");
 #endif /* ndef FEATURE_STATISTICS */
@@ -1412,61 +1461,45 @@ char *dump_map(const struct map *the_map)
 }
 
 
-#ifdef FEATURE_STATISTICS
 /*********************************************************************
  *
- * Function    :  add_stats
+ * Function    :  cgi_robots_txt
  *
- * Description :  Add the blocking statistics to a given map.
+ * Description :  CGI function to return "/robots.txt".
  *
  * Parameters  :
- *          1  :  exports = map to write to.
+ *           1 :  csp = Current client state (buffers, headers, etc...)
+ *           2 :  rsp = http_response data structure for output
+ *           3 :  parameters = map of cgi parameters
  *
- * Returns     :  pointer to extended map
+ * CGI Parameters : None
+ *
+ * Returns     :  0
  *
  *********************************************************************/
-struct map *add_stats(struct map *exports)
+int cgi_robots_txt(struct client_state *csp, struct http_response *rsp,
+                   struct map *parameters)
 {
-   float perc_rej;   /* Percentage of http requests rejected */
-   char buf[1000];
-   int local_urls_read     = urls_read;
-   int local_urls_rejected = urls_rejected;
-
-   /*
-    * Need to alter the stats not to include the fetch of this
-    * page.
-    *
-    * Can't do following thread safely! doh!
-    *
-    * urls_read--;
-    * urls_rejected--; * This will be incremented subsequently *
-    */
-
-   if (local_urls_read == 0)
-   {
-      map_block_killer(exports, "have-stats");
-   }
-   else
-   {
-      map_block_killer(exports, "have-no-stats");
-
-      perc_rej = (float)local_urls_rejected * 100.0F /
-            (float)local_urls_read;
+   char buf[100];
 
-      sprintf(buf, "%d", local_urls_read);
-      map(exports, "requests-received", 1, buf, 1);
+   rsp->body = strdup(
+      "# This is the Internet Junkbuster control interface.\n"
+      "# It isn't very useful to index it, and you're likely to break stuff.\n"
+      "# So go away!\n"
+      "\n"
+      "User-agent: *\n"
+      "Disallow: /\n"
+      "\n");
 
-      sprintf(buf, "%d", local_urls_rejected);
-      map(exports, "requests-blocked", 1, buf, 1);
+   enlist_unique(rsp->headers, "Content-Type: text/plain", 13);
 
-      sprintf(buf, "%6.2f", perc_rej);
-      map(exports, "percent-blocked", 1, buf, 1);
-   }
+   rsp->is_static = 1;
 
-   return(exports);
+   get_http_time(7 * 24 * 60 * 60, buf); /* 7 days into future */
+   enlist_unique_header(rsp->headers, "Expires", buf);
 
+   return 0;
 }
-#endif /* def FEATURE_STATISTICS */
 
 
 /*

diff --git a/cgi.h b/cgi.h
index 5cc92d7..4f155d6 100644 (file)
--- a/cgi.h
+++ b/cgi.h
@@ -1,6 +1,6 @@
 #ifndef CGI_H_INCLUDED
 #define CGI_H_INCLUDED
-#define CGI_H_VERSION "$Id: cgi.h,v 1.13 2001/09/16 11:00:10 jongfoster Exp $"
+#define CGI_H_VERSION "$Id: cgi.h,v 1.14 2001/09/16 11:38:02 jongfoster Exp $"
 /*********************************************************************
  *
  * File        :  $Source: /cvsroot/ijbswa/current/cgi.h,v $
@@ -38,6 +38,14 @@
  *
  * Revisions   :
  *    $Log: cgi.h,v $
+ *    Revision 1.14  2001/09/16 11:38:02  jongfoster
+ *    Splitting fill_template() into 2 functions:
+ *    template_load() loads the file
+ *    template_fill() performs the PCRS regexps.
+ *    This is because the CGI edit interface has a "table row"
+ *    template which is used many times in the page - this
+ *    change means it's only loaded from disk once.
+ *
  *    Revision 1.13  2001/09/16 11:00:10  jongfoster
  *    New function alloc_http_response, for symmetry with free_http_response
  *
@@ -105,19 +113,29 @@ extern struct map *parse_cgi_parameters(char *argstring);
 /*
  * CGI functions
  */
-extern int cgi_show_version(struct client_state *csp, struct http_response *rsp,
-                            struct map *parameters);
-extern int cgi_default(struct client_state *csp, struct http_response *rsp,
-                       struct map *parameters);
-extern int cgi_show_status(struct client_state *csp, struct http_response *rsp,
-                           struct map *parameters);
-extern int cgi_show_url_info(struct client_state *csp, struct http_response *rsp,
-                             struct map *parameters);
-extern int cgi_send_banner(struct client_state *csp, struct http_response *rsp,
-                                 struct map *parameters);
+extern int cgi_default             (struct client_state *csp,
+                                    struct http_response *rsp,
+                                    struct map *parameters);
+extern int cgi_robots_txt          (struct client_state *csp,
+                                    struct http_response *rsp,
+                                    struct map *parameters);
+extern int cgi_send_banner         (struct client_state *csp,
+                                    struct http_response *rsp,
+                                    struct map *parameters);
+extern int cgi_show_status         (struct client_state *csp,
+                                    struct http_response *rsp,
+                                    struct map *parameters);
+extern int cgi_show_url_info       (struct client_state *csp,
+                                    struct http_response *rsp,
+                                    struct map *parameters);
+extern int cgi_show_version        (struct client_state *csp,
+                                    struct http_response *rsp,
+                                    struct map *parameters);
 
 /* Not exactly a CGI */
-extern struct http_response *error_response(struct client_state *csp, const char *template, int err);
+extern struct http_response * error_response(struct client_state *csp,
+                                             const char *templatename,
+                                             int err);
 
 /*
  * CGI support functions
@@ -141,10 +159,6 @@ extern void template_fill(char ** template_ptr, struct map *exports);
 extern char *make_menu(const char *self);
 extern char *dump_map(const struct map *map);
 
-#ifdef FEATURE_STATISTICS
-extern struct map *add_stats(struct map *exports);
-#endif /* def FEATURE_STATISTICS */
-
 /*
  * Some images.
  */