Merge branch 'master' of ssh://git.privoxy.org:23/git/privoxy
[privoxy.git] / jcc.c
1 /*********************************************************************
2  *
3  * File        :  $Source: /cvsroot/ijbswa/current/jcc.c,v $
4  *
5  * Purpose     :  Main file.  Contains main() method, main loop, and
6  *                the main connection-handling function.
7  *
8  * Copyright   :  Written by and Copyright (C) 2001-2020 the
9  *                Privoxy team. https://www.privoxy.org/
10  *
11  *                Based on the Internet Junkbuster originally written
12  *                by and Copyright (C) 1997 Anonymous Coders and
13  *                Junkbusters Corporation.  http://www.junkbusters.com
14  *
15  *                This program is free software; you can redistribute it
16  *                and/or modify it under the terms of the GNU General
17  *                Public License as published by the Free Software
18  *                Foundation; either version 2 of the License, or (at
19  *                your option) any later version.
20  *
21  *                This program is distributed in the hope that it will
22  *                be useful, but WITHOUT ANY WARRANTY; without even the
23  *                implied warranty of MERCHANTABILITY or FITNESS FOR A
24  *                PARTICULAR PURPOSE.  See the GNU General Public
25  *                License for more details.
26  *
27  *                The GNU General Public License should be included with
28  *                this file.  If not, you can view it at
29  *                http://www.gnu.org/copyleft/gpl.html
30  *                or write to the Free Software Foundation, Inc., 59
31  *                Temple Place - Suite 330, Boston, MA  02111-1307, USA.
32  *
33  *********************************************************************/
34
35
36 #include "config.h"
37
38 #include <stdio.h>
39 #include <sys/types.h>
40 #include <stdlib.h>
41 #include <string.h>
42 #include <signal.h>
43 #include <fcntl.h>
44 #include <errno.h>
45 #include <assert.h>
46
47 #ifdef _WIN32
48 # ifndef FEATURE_PTHREAD
49 #  ifndef STRICT
50 #   define STRICT
51 #  endif
52 #  include <winsock2.h>
53 #  include <windows.h>
54 #  include <process.h>
55 # endif /* ndef FEATURE_PTHREAD */
56
57 # include "win32.h"
58 # ifndef _WIN_CONSOLE
59 #  include "w32log.h"
60 # endif /* ndef _WIN_CONSOLE */
61 # include "w32svrapi.h"
62
63 #else /* ifndef _WIN32 */
64
65 # include <unistd.h>
66 # include <sys/wait.h>
67 # include <sys/time.h>
68 # include <sys/stat.h>
69 # include <sys/ioctl.h>
70
71 #ifdef sun
72 #include <sys/termios.h>
73 #endif /* sun */
74
75 #ifdef unix
76 #include <pwd.h>
77 #include <grp.h>
78 #endif
79
80 # include <signal.h>
81
82 # ifdef __BEOS__
83 #  include <socket.h>  /* BeOS has select() for sockets only. */
84 #  include <OS.h>      /* declarations for threads and stuff. */
85 # endif
86
87 #ifdef HAVE_POLL
88 #ifdef __GLIBC__
89 #include <sys/poll.h>
90 #else
91 #include <poll.h>
92 #endif /* def __GLIBC__ */
93 #endif /* HAVE_POLL */
94
95 #endif
96
97 #include "project.h"
98 #include "list.h"
99 #include "jcc.h"
100 #ifdef FEATURE_HTTPS_INSPECTION
101 #include "ssl.h"
102 #endif
103 #include "filters.h"
104 #include "loaders.h"
105 #include "parsers.h"
106 #include "miscutil.h"
107 #include "errlog.h"
108 #include "jbsockets.h"
109 #include "gateway.h"
110 #include "actions.h"
111 #include "cgi.h"
112 #include "loadcfg.h"
113 #include "urlmatch.h"
114 #ifdef FEATURE_CLIENT_TAGS
115 #include "client-tags.h"
116 #endif
117
118 int daemon_mode = 1;
119 struct client_states clients[1];
120 struct file_list     files[1];
121
122 #ifdef FEATURE_STATISTICS
123 int urls_read     = 0;     /* total nr of urls read inc rejected */
124 int urls_rejected = 0;     /* total nr of urls rejected */
125 #endif /* def FEATURE_STATISTICS */
126
127 #ifdef FEATURE_GRACEFUL_TERMINATION
128 int g_terminate = 0;
129 #endif
130
131 #if !defined(_WIN32)
132 static void sig_handler(int the_signal);
133 #endif
134 static int client_protocol_is_unsupported(struct client_state *csp, char *req);
135 static jb_err get_request_destination_elsewhere(struct client_state *csp, struct list *headers);
136 static jb_err get_server_headers(struct client_state *csp);
137 static const char *crunch_reason(const struct http_response *rsp);
138 static void send_crunch_response(struct client_state *csp, struct http_response *rsp);
139 static char *get_request_line(struct client_state *csp);
140 static jb_err receive_client_request(struct client_state *csp);
141 static jb_err parse_client_request(struct client_state *csp);
142 static void build_request_line(struct client_state *csp, const struct forward_spec *fwd, char **request_line);
143 static jb_err change_request_destination(struct client_state *csp);
144 static void handle_established_connection(struct client_state *csp);
145 static void chat(struct client_state *csp);
146 static void serve(struct client_state *csp);
147 #if !defined(_WIN32) || defined(_WIN_CONSOLE)
148 static void usage(const char *myname);
149 #endif
150 static void initialize_mutexes(void);
151 static jb_socket bind_port_helper(const char *haddr, int hport, int backlog);
152 static void bind_ports_helper(struct configuration_spec *config, jb_socket sockets[]);
153 static void close_ports_helper(jb_socket sockets[]);
154 static void listen_loop(void);
155 static void serve(struct client_state *csp);
156
157 #ifdef __BEOS__
158 static int32 server_thread(void *data);
159 #endif /* def __BEOS__ */
160
161 #ifdef _WIN32
162 #define sleep(N)  Sleep(((N) * 1000))
163 #endif
164
165 #ifdef FUZZ
166 int process_fuzzed_input(char *fuzz_input_type, char *fuzz_input_file);
167 void show_fuzz_usage(const char *name);
168 #endif
169
170 #ifdef MUTEX_LOCKS_AVAILABLE
171 /*
172  * XXX: Does the locking stuff really belong in this file?
173  */
174 privoxy_mutex_t log_mutex;
175 privoxy_mutex_t log_init_mutex;
176 privoxy_mutex_t connection_reuse_mutex;
177
178 #ifdef FEATURE_HTTPS_INSPECTION
179 privoxy_mutex_t certificate_mutex;
180 privoxy_mutex_t ssl_init_mutex;
181 #endif
182
183 #ifdef FEATURE_EXTERNAL_FILTERS
184 privoxy_mutex_t external_filter_mutex;
185 #endif
186 #ifdef FEATURE_CLIENT_TAGS
187 privoxy_mutex_t client_tags_mutex;
188 #endif
189 #ifdef FEATURE_EXTENDED_STATISTICS
190 privoxy_mutex_t filter_statistics_mutex;
191 privoxy_mutex_t block_statistics_mutex;
192 #endif
193
194 #if !defined(HAVE_GETHOSTBYADDR_R) || !defined(HAVE_GETHOSTBYNAME_R)
195 privoxy_mutex_t resolver_mutex;
196 #endif /* !defined(HAVE_GETHOSTBYADDR_R) || !defined(HAVE_GETHOSTBYNAME_R) */
197
198 #ifndef HAVE_GMTIME_R
199 privoxy_mutex_t gmtime_mutex;
200 #endif /* ndef HAVE_GMTIME_R */
201
202 #ifndef HAVE_LOCALTIME_R
203 privoxy_mutex_t localtime_mutex;
204 #endif /* ndef HAVE_GMTIME_R */
205
206 #if !defined(HAVE_ARC4RANDOM) && !defined(HAVE_RANDOM)
207 privoxy_mutex_t rand_mutex;
208 #endif /* !defined(HAVE_ARC4RANDOM) && !defined(HAVE_RANDOM) */
209
210 #endif /* def MUTEX_LOCKS_AVAILABLE */
211
212 #if defined(unix)
213 const char *basedir = NULL;
214 const char *pidfile = NULL;
215 static int received_hup_signal = 0;
216 #endif /* defined unix */
217
218 /* HTTP snipplets. */
219 static const char CSUCCEED[] =
220    "HTTP/1.1 200 Connection established\r\n\r\n";
221
222 static const char CHEADER[] =
223    "HTTP/1.1 400 Invalid header received from client\r\n"
224    "Content-Type: text/plain\r\n"
225    "Connection: close\r\n\r\n"
226    "Invalid header received from client.\r\n";
227
228 static const char FTP_RESPONSE[] =
229    "HTTP/1.1 400 Invalid request received from client\r\n"
230    "Content-Type: text/plain\r\n"
231    "Connection: close\r\n\r\n"
232    "Invalid request. Privoxy doesn't support FTP.\r\n";
233
234 static const char GOPHER_RESPONSE[] =
235    "HTTP/1.1 400 Invalid request received from client\r\n"
236    "Content-Type: text/plain\r\n"
237    "Connection: close\r\n\r\n"
238    "Invalid request. Privoxy doesn't support gopher.\r\n";
239
240 /* XXX: should be a template */
241 static const char MISSING_DESTINATION_RESPONSE[] =
242    "HTTP/1.1 400 Bad request received from client\r\n"
243    "Content-Type: text/plain\r\n"
244    "Connection: close\r\n\r\n"
245    "Bad request. Privoxy was unable to extract the destination.\r\n";
246
247 /* XXX: should be a template */
248 static const char INVALID_SERVER_HEADERS_RESPONSE[] =
249    "HTTP/1.1 502 Server or forwarder response invalid\r\n"
250    "Content-Type: text/plain\r\n"
251    "Connection: close\r\n\r\n"
252    "Bad response. The server or forwarder response doesn't look like HTTP.\r\n";
253
254 /* XXX: should be a template */
255 static const char MESSED_UP_REQUEST_RESPONSE[] =
256    "HTTP/1.1 400 Malformed request after rewriting\r\n"
257    "Content-Type: text/plain\r\n"
258    "Connection: close\r\n\r\n"
259    "Bad request. Messed up with header filters.\r\n";
260
261 static const char TOO_MANY_CONNECTIONS_RESPONSE[] =
262    "HTTP/1.1 503 Too many open connections\r\n"
263    "Content-Type: text/plain\r\n"
264    "Connection: close\r\n\r\n"
265    "Maximum number of open connections reached.\r\n";
266
267 static const char CLIENT_CONNECTION_TIMEOUT_RESPONSE[] =
268    "HTTP/1.1 504 Connection timeout\r\n"
269    "Content-Type: text/plain\r\n"
270    "Connection: close\r\n\r\n"
271    "The connection timed out because the client request didn't arrive in time.\r\n";
272
273 static const char CLIENT_BODY_PARSE_ERROR_RESPONSE[] =
274    "HTTP/1.1 400 Failed reading client body\r\n"
275    "Content-Type: text/plain\r\n"
276    "Connection: close\r\n\r\n"
277    "Failed parsing or buffering the chunk-encoded client body.\r\n";
278
279 static const char UNSUPPORTED_CLIENT_EXPECTATION_ERROR_RESPONSE[] =
280    "HTTP/1.1 417 Expecting too much\r\n"
281    "Content-Type: text/plain\r\n"
282    "Connection: close\r\n\r\n"
283    "Privoxy detected an unsupported Expect header value.\r\n";
284
285 /* A function to crunch a response */
286 typedef struct http_response *(*crunch_func_ptr)(struct client_state *);
287
288 /* Crunch function flags */
289 #define CF_NO_FLAGS        0
290 /* Cruncher applies to forced requests as well */
291 #define CF_IGNORE_FORCE    1
292 /* Crunched requests are counted for the block statistics */
293 #define CF_COUNT_AS_REJECT 2
294
295 /* A crunch function and its flags */
296 struct cruncher
297 {
298    const crunch_func_ptr cruncher;
299    const int flags;
300 };
301
302 static int crunch_response_triggered(struct client_state *csp, const struct cruncher crunchers[]);
303
304 /* Complete list of cruncher functions */
305 static const struct cruncher crunchers_all[] = {
306    { direct_response, CF_COUNT_AS_REJECT|CF_IGNORE_FORCE},
307    { block_url,       CF_COUNT_AS_REJECT },
308 #ifdef FEATURE_TRUST
309    { trust_url,       CF_COUNT_AS_REJECT },
310 #endif /* def FEATURE_TRUST */
311    { redirect_url,    CF_NO_FLAGS  },
312    { dispatch_cgi,    CF_IGNORE_FORCE},
313    { NULL,            0 }
314 };
315
316 /* Light version, used after tags are applied */
317 static const struct cruncher crunchers_light[] = {
318    { block_url,       CF_COUNT_AS_REJECT },
319    { redirect_url,    CF_NO_FLAGS },
320    { NULL,            0 }
321 };
322
323
324 /*
325  * XXX: Don't we really mean
326  *
327  * #if defined(unix)
328  *
329  * here?
330  */
331 #if !defined(_WIN32)
332 /*********************************************************************
333  *
334  * Function    :  sig_handler
335  *
336  * Description :  Signal handler for different signals.
337  *                Exit gracefully on TERM and INT
338  *                or set a flag that will cause the errlog
339  *                to be reopened by the main thread on HUP.
340  *
341  * Parameters  :
342  *          1  :  the_signal = the signal cause this function to call
343  *
344  * Returns     :  -
345  *
346  *********************************************************************/
347 static void sig_handler(int the_signal)
348 {
349    switch(the_signal)
350    {
351       case SIGTERM:
352       case SIGINT:
353          log_error(LOG_LEVEL_INFO, "exiting by signal %d .. bye", the_signal);
354 #if defined(unix)
355          if (pidfile)
356          {
357             unlink(pidfile);
358          }
359 #endif /* unix */
360          exit(the_signal);
361          break;
362
363       case SIGHUP:
364 #if defined(unix)
365          received_hup_signal = 1;
366 #endif
367          break;
368
369       default:
370          /*
371           * We shouldn't be here, unless we catch signals
372           * in main() that we can't handle here!
373           */
374          log_error(LOG_LEVEL_FATAL, "sig_handler: exiting on unexpected signal %d", the_signal);
375    }
376    return;
377
378 }
379 #endif
380
381
382 /*********************************************************************
383  *
384  * Function    :  get_write_delay
385  *
386  * Description :  Parse the delay-response parameter.
387  *
388  * Parameters  :
389  *          1  :  csp = Current client state (buffers, headers, etc...)
390  *
391  * Returns     :  Number of milliseconds to delay writes.
392  *
393  *********************************************************************/
394 static unsigned int get_write_delay(const struct client_state *csp)
395 {
396    unsigned int delay;
397    char *endptr;
398    char *newval;
399
400    if ((csp->action->flags & ACTION_DELAY_RESPONSE) == 0)
401    {
402       return 0;
403    }
404    newval = csp->action->string[ACTION_STRING_DELAY_RESPONSE];
405
406    delay = (unsigned)strtol(newval, &endptr, 0);
407    if (*endptr != '\0')
408    {
409       log_error(LOG_LEVEL_FATAL,
410          "Invalid delay-response{} parameter: '%s'", newval);
411    }
412
413    return delay;
414
415 }
416
417
418 /*********************************************************************
419  *
420  * Function    :  client_protocol_is_unsupported
421  *
422  * Description :  Checks if the client used a known unsupported
423  *                protocol and deals with it by sending an error
424  *                response.
425  *
426  * Parameters  :
427  *          1  :  csp = Current client state (buffers, headers, etc...)
428  *          2  :  req = the first request line send by the client
429  *
430  * Returns     :  TRUE if an error response has been generated, or
431  *                FALSE if the request doesn't look invalid.
432  *
433  *********************************************************************/
434 static int client_protocol_is_unsupported(struct client_state *csp, char *req)
435 {
436    /*
437     * If it's a FTP or gopher request, we don't support it.
438     *
439     * These checks are better than nothing, but they might
440     * not work in all configurations and some clients might
441     * have problems digesting the answer.
442     *
443     * They should, however, never cause more problems than
444     * Privoxy's old behaviour (returning the misleading HTML
445     * error message:
446     *
447     * "Could not resolve http://(ftp|gopher)://example.org").
448     */
449    if (!strncmpic(req, "GET ftp://", 10) || !strncmpic(req, "GET gopher://", 13))
450    {
451       const char *response = NULL;
452       const char *protocol = NULL;
453
454       if (!strncmpic(req, "GET ftp://", 10))
455       {
456          response = FTP_RESPONSE;
457          protocol = "FTP";
458       }
459       else
460       {
461          response = GOPHER_RESPONSE;
462          protocol = "GOPHER";
463       }
464       log_error(LOG_LEVEL_ERROR,
465          "%s tried to use Privoxy as %s proxy: %s",
466          csp->ip_addr_str, protocol, req);
467       log_error(LOG_LEVEL_CLF,
468          "%s - - [%T] \"%s\" 400 0", csp->ip_addr_str, req);
469       freez(req);
470
471 #ifdef FEATURE_HTTPS_INSPECTION
472       if (client_use_ssl(csp))
473       {
474          ssl_send_data_delayed(&(csp->ssl_client_attr),
475             (const unsigned char *)response, strlen(response),
476             get_write_delay(csp));
477       }
478       else
479 #endif
480       {
481          write_socket_delayed(csp->cfd, response, strlen(response),
482             get_write_delay(csp));
483       }
484
485       return TRUE;
486    }
487
488    return FALSE;
489 }
490
491
492 /*********************************************************************
493  *
494  * Function    :  client_has_unsupported_expectations
495  *
496  * Description :  Checks if the client used an unsupported expectation
497  *                in which case an error message is delivered.
498  *
499  * Parameters  :
500  *          1  :  csp = Current client state (buffers, headers, etc...)
501  *
502  * Returns     :  TRUE if an error response has been generated, or
503  *                FALSE if the request doesn't look invalid.
504  *
505  *********************************************************************/
506 static int client_has_unsupported_expectations(const struct client_state *csp)
507 {
508    if ((csp->flags & CSP_FLAG_UNSUPPORTED_CLIENT_EXPECTATION))
509    {
510       log_error(LOG_LEVEL_ERROR,
511          "Rejecting request from client %s with unsupported Expect header value",
512          csp->ip_addr_str);
513       log_error(LOG_LEVEL_CLF,
514          "%s - - [%T] \"%s\" 417 0", csp->ip_addr_str, csp->http->cmd);
515       write_socket_delayed(csp->cfd,
516          UNSUPPORTED_CLIENT_EXPECTATION_ERROR_RESPONSE,
517          strlen(UNSUPPORTED_CLIENT_EXPECTATION_ERROR_RESPONSE),
518          get_write_delay(csp));
519
520       return TRUE;
521    }
522
523    return FALSE;
524
525 }
526
527
528 /*********************************************************************
529  *
530  * Function    :  get_request_destination_elsewhere
531  *
532  * Description :  If the client's request was redirected into
533  *                Privoxy without the client's knowledge,
534  *                the request line lacks the destination host.
535  *
536  *                This function tries to get it elsewhere,
537  *                provided accept-intercepted-requests is enabled.
538  *
539  *                "Elsewhere" currently only means "Host: header",
540  *                but in the future we may ask the redirecting
541  *                packet filter to look the destination up.
542  *
543  *                If the destination stays unknown, an error
544  *                response is send to the client and headers
545  *                are freed so that chat() can return directly.
546  *
547  * Parameters  :
548  *          1  :  csp = Current client state (buffers, headers, etc...)
549  *          2  :  headers = a header list
550  *
551  * Returns     :  JB_ERR_OK if the destination is now known, or
552  *                JB_ERR_PARSE if it isn't.
553  *
554  *********************************************************************/
555 static jb_err get_request_destination_elsewhere(struct client_state *csp, struct list *headers)
556 {
557    if (!(csp->config->feature_flags & RUNTIME_FEATURE_ACCEPT_INTERCEPTED_REQUESTS))
558    {
559       log_error(LOG_LEVEL_ERROR, "%s's request: \'%s\' is invalid."
560          " Privoxy isn't configured to accept intercepted requests.",
561          csp->ip_addr_str, csp->http->cmd);
562       /* XXX: Use correct size */
563       log_error(LOG_LEVEL_CLF, "%s - - [%T] \"%s\" 400 0",
564          csp->ip_addr_str, csp->http->cmd);
565
566       write_socket_delayed(csp->cfd, CHEADER, strlen(CHEADER),
567          get_write_delay(csp));
568       destroy_list(headers);
569
570       return JB_ERR_PARSE;
571    }
572    else if (JB_ERR_OK == get_destination_from_headers(headers, csp->http))
573    {
574       /* Split the domain we just got for pattern matching */
575       init_domain_components(csp->http);
576
577       return JB_ERR_OK;
578    }
579    else
580    {
581       /* We can't work without destination. Go spread the news.*/
582
583       /* XXX: Use correct size */
584       log_error(LOG_LEVEL_CLF, "%s - - [%T] \"%s\" 400 0",
585          csp->ip_addr_str, csp->http->cmd);
586       log_error(LOG_LEVEL_ERROR,
587          "Privoxy was unable to get the destination for %s's request: %s",
588          csp->ip_addr_str, csp->http->cmd);
589
590       write_socket_delayed(csp->cfd, MISSING_DESTINATION_RESPONSE,
591          strlen(MISSING_DESTINATION_RESPONSE), get_write_delay(csp));
592       destroy_list(headers);
593
594       return JB_ERR_PARSE;
595    }
596    /*
597     * TODO: If available, use PF's ioctl DIOCNATLOOK as last resort
598     * to get the destination IP address, use it as host directly
599     * or do a reverse DNS lookup first.
600     */
601 }
602
603
604 /*********************************************************************
605  *
606  * Function    :  get_server_headers
607  *
608  * Description :  Parses server headers in iob and fills them
609  *                into csp->headers so that they can later be
610  *                handled by sed().
611  *
612  * Parameters  :
613  *          1  :  csp = Current client state (buffers, headers, etc...)
614  *
615  * Returns     :  JB_ERR_OK if everything went fine, or
616  *                JB_ERR_PARSE if the headers were incomplete.
617  *
618  *********************************************************************/
619 static jb_err get_server_headers(struct client_state *csp)
620 {
621    int continue_hack_in_da_house = 0;
622    char * header;
623
624    while (((header = get_header(csp->iob)) != NULL) || continue_hack_in_da_house)
625    {
626       if (header == NULL)
627       {
628          /*
629           * continue hack in da house. Ignore the ending of
630           * this head and continue enlisting header lines.
631           * The reason is described below.
632           */
633          enlist(csp->headers, "");
634          continue_hack_in_da_house = 0;
635          continue;
636       }
637       else if (0 == strncmpic(header, "HTTP/1.1 100", 12))
638       {
639          /*
640           * It's a bodyless continue response, don't
641           * stop header parsing after reaching its end.
642           *
643           * As a result Privoxy will concatenate the
644           * next response's head and parse and deliver
645           * the headers as if they belonged to one request.
646           *
647           * The client will separate them because of the
648           * empty line between them.
649           *
650           * XXX: What we're doing here is clearly against
651           * the intended purpose of the continue header,
652           * and under some conditions (HTTP/1.0 client request)
653           * it's a standard violation.
654           *
655           * Anyway, "sort of against the spec" is preferable
656           * to "always getting confused by Continue responses"
657           * (Privoxy's behaviour before this hack was added)
658           */
659          log_error(LOG_LEVEL_HEADER, "Continue hack in da house.");
660          continue_hack_in_da_house = 1;
661       }
662       else if (*header == '\0')
663       {
664          /*
665           * If the header is empty, but the Continue hack
666           * isn't active, we can assume that we reached the
667           * end of the buffer before we hit the end of the
668           * head.
669           *
670           * Inform the caller an let it decide how to handle it.
671           */
672          return JB_ERR_PARSE;
673       }
674
675       if (JB_ERR_MEMORY == enlist(csp->headers, header))
676       {
677          /*
678           * XXX: Should we quit the request and return a
679           * out of memory error page instead?
680           */
681          log_error(LOG_LEVEL_ERROR,
682             "Out of memory while enlisting server headers. %s lost.",
683             header);
684       }
685       freez(header);
686    }
687
688    return JB_ERR_OK;
689 }
690
691
692 /*********************************************************************
693  *
694  * Function    :  crunch_reason
695  *
696  * Description :  Translates the crunch reason code into a string.
697  *
698  * Parameters  :
699  *          1  :  rsp = a http_response
700  *
701  * Returns     :  A string with the crunch reason or an error description.
702  *
703  *********************************************************************/
704 static const char *crunch_reason(const struct http_response *rsp)
705 {
706    char * reason = NULL;
707
708    assert(rsp != NULL);
709    if (rsp == NULL)
710    {
711       return "Internal error while searching for crunch reason";
712    }
713
714    switch (rsp->crunch_reason)
715    {
716       case UNSUPPORTED:
717          reason = "Unsupported HTTP feature";
718          break;
719       case BLOCKED:
720          reason = "Blocked";
721          break;
722       case UNTRUSTED:
723          reason = "Untrusted";
724          break;
725       case REDIRECTED:
726          reason = "Redirected";
727          break;
728       case CGI_CALL:
729          reason = "CGI Call";
730          break;
731       case NO_SUCH_DOMAIN:
732          reason = "DNS failure";
733          break;
734       case FORWARDING_FAILED:
735          reason = "Forwarding failed";
736          break;
737       case CONNECT_FAILED:
738          reason = "Connection failure";
739          break;
740       case OUT_OF_MEMORY:
741          reason = "Out of memory (may mask other reasons)";
742          break;
743       case CONNECTION_TIMEOUT:
744          reason = "Connection timeout";
745          break;
746       case NO_SERVER_DATA:
747          reason = "No server data received";
748          break;
749       default:
750          reason = "No reason recorded";
751          break;
752    }
753
754    return reason;
755 }
756
757
758 /*********************************************************************
759  *
760  * Function    :  log_applied_actions
761  *
762  * Description :  Logs the applied actions if LOG_LEVEL_ACTIONS is
763  *                enabled.
764  *
765  * Parameters  :
766  *          1  :  actions = Current action spec to log
767  *
768  * Returns     :  Nothing.
769  *
770  *********************************************************************/
771 static void log_applied_actions(const struct current_action_spec *actions)
772 {
773    /*
774     * The conversion to text requires lots of memory allocations so
775     * we only do the conversion if the user is actually interested.
776     */
777    if (debug_level_is_enabled(LOG_LEVEL_ACTIONS))
778    {
779       char *actions_as_text = actions_to_line_of_text(actions);
780       log_error(LOG_LEVEL_ACTIONS, "%s", actions_as_text);
781       freez(actions_as_text);
782    }
783 }
784
785
786 /*********************************************************************
787  *
788  * Function    :  send_crunch_response
789  *
790  * Description :  Delivers already prepared response for
791  *                intercepted requests, logs the interception
792  *                and frees the response.
793  *
794  * Parameters  :
795  *          1  :  csp = Current client state (buffers, headers, etc...)
796  *          2  :  rsp = Fully prepared response. Will be freed on exit.
797  *
798  * Returns     :  Nothing.
799  *
800  *********************************************************************/
801 static void send_crunch_response(struct client_state *csp, struct http_response *rsp)
802 {
803       const struct http_request *http = csp->http;
804       char status_code[4];
805
806       assert(rsp != NULL);
807       assert(rsp->head != NULL);
808
809       if (rsp == NULL)
810       {
811          log_error(LOG_LEVEL_FATAL, "NULL response in send_crunch_response.");
812       }
813
814       /*
815        * Extract the status code from the actual head
816        * that will be send to the client. It is the only
817        * way to get it right for all requests, including
818        * the fixed ones for out-of-memory problems.
819        *
820        * A head starts like this: 'HTTP/1.1 200...'
821        *                           0123456789|11
822        *                                     10
823        */
824       status_code[0] = rsp->head[9];
825       status_code[1] = rsp->head[10];
826       status_code[2] = rsp->head[11];
827       status_code[3] = '\0';
828
829       /* Log that the request was crunched and why. */
830       log_applied_actions(csp->action);
831 #ifdef FEATURE_HTTPS_INSPECTION
832       if (client_use_ssl(csp))
833       {
834          log_error(LOG_LEVEL_CRUNCH, "%s: https://%s%s", crunch_reason(rsp),
835             http->hostport, http->path);
836          log_error(LOG_LEVEL_CLF, "%s - - [%T] \"%s https://%s%s %s\" %s %lu",
837             csp->ip_addr_str, http->gpc, http->hostport, http->path,
838             http->version, status_code, rsp->content_length);
839       }
840       else
841 #endif
842       {
843          log_error(LOG_LEVEL_CRUNCH, "%s: %s", crunch_reason(rsp), http->url);
844          log_error(LOG_LEVEL_CLF, "%s - - [%T] \"%s\" %s %lu",
845             csp->ip_addr_str, http->ocmd, status_code, rsp->content_length);
846       }
847       /* Write the answer to the client */
848 #ifdef FEATURE_HTTPS_INSPECTION
849       if (client_use_ssl(csp))
850       {
851          if ((ssl_send_data_delayed(&(csp->ssl_client_attr),
852                 (const unsigned char *)rsp->head, rsp->head_length,
853                 get_write_delay(csp)) < 0)
854           || (ssl_send_data_delayed(&(csp->ssl_client_attr),
855                 (const unsigned char *)rsp->body, rsp->content_length,
856                 get_write_delay(csp)) < 0))
857          {
858             /* There is nothing we can do about it. */
859             log_error(LOG_LEVEL_CONNECT, "Couldn't deliver the error message "
860                "for https://%s%s through client socket %d using TLS/SSL",
861                http->hostport, http->url, csp->cfd);
862          }
863       }
864       else
865 #endif
866       {
867          if (write_socket_delayed(csp->cfd, rsp->head, rsp->head_length,
868                 get_write_delay(csp))
869           || write_socket_delayed(csp->cfd, rsp->body, rsp->content_length,
870                 get_write_delay(csp)))
871          {
872             /* There is nothing we can do about it. */
873             log_error(LOG_LEVEL_CONNECT,
874                "Couldn't deliver the error message for %s through client socket %d: %E",
875                http->url, csp->cfd);
876          }
877       }
878
879       /* Clean up and return */
880       if (cgi_error_memory() != rsp)
881       {
882          free_http_response(rsp);
883       }
884       return;
885 }
886
887
888 /*********************************************************************
889  *
890  * Function    :  crunch_response_triggered
891  *
892  * Description :  Checks if the request has to be crunched,
893  *                and delivers the crunch response if necessary.
894  *
895  * Parameters  :
896  *          1  :  csp = Current client state (buffers, headers, etc...)
897  *          2  :  crunchers = list of cruncher functions to run
898  *
899  * Returns     :  TRUE if the request was answered with a crunch response
900  *                FALSE otherwise.
901  *
902  *********************************************************************/
903 static int crunch_response_triggered(struct client_state *csp, const struct cruncher crunchers[])
904 {
905    struct http_response *rsp = NULL;
906    const struct cruncher *c;
907
908    /*
909     * If CGI request crunching is disabled,
910     * check the CGI dispatcher out of order to
911     * prevent unintentional blocks or redirects.
912     */
913    if (!(csp->config->feature_flags & RUNTIME_FEATURE_CGI_CRUNCHING)
914        && (NULL != (rsp = dispatch_cgi(csp))))
915    {
916       /* Deliver, log and free the interception response. */
917       send_crunch_response(csp, rsp);
918       csp->flags |= CSP_FLAG_CRUNCHED;
919       return TRUE;
920    }
921
922    for (c = crunchers; c->cruncher != NULL; c++)
923    {
924       /*
925        * Check the cruncher if either Privoxy is toggled
926        * on and the request isn't forced, or if the cruncher
927        * applies to forced requests as well.
928        */
929       if (((csp->flags & CSP_FLAG_TOGGLED_ON) &&
930           !(csp->flags & CSP_FLAG_FORCED)) ||
931           (c->flags & CF_IGNORE_FORCE))
932       {
933          rsp = c->cruncher(csp);
934          if (NULL != rsp)
935          {
936             /* Deliver, log and free the interception response. */
937             send_crunch_response(csp, rsp);
938             csp->flags |= CSP_FLAG_CRUNCHED;
939 #ifdef FEATURE_STATISTICS
940             if (c->flags & CF_COUNT_AS_REJECT)
941             {
942                csp->flags |= CSP_FLAG_REJECTED;
943             }
944 #endif /* def FEATURE_STATISTICS */
945
946             return TRUE;
947          }
948       }
949    }
950
951    return FALSE;
952 }
953
954
955 /*********************************************************************
956  *
957  * Function    :  build_request_line
958  *
959  * Description :  Builds the HTTP request line.
960  *
961  *                If a HTTP forwarder is used it expects the whole URL,
962  *                web servers only get the path.
963  *
964  * Parameters  :
965  *          1  :  csp = Current client state (buffers, headers, etc...)
966  *          2  :  fwd = The forwarding spec used for the request
967  *                XXX: Should use http->fwd instead.
968  *          3  :  request_line = The old request line which will be replaced.
969  *
970  * Returns     :  Nothing. Terminates in case of memory problems.
971  *
972  *********************************************************************/
973 static void build_request_line(struct client_state *csp, const struct forward_spec *fwd, char **request_line)
974 {
975    struct http_request *http = csp->http;
976
977    /*
978     * Downgrade http version from 1.1 to 1.0
979     * if +downgrade action applies.
980     */
981    if ((csp->action->flags & ACTION_DOWNGRADE)
982      && (!strcmpic(http->version, "HTTP/1.1")))
983    {
984       freez(http->version);
985       http->version = strdup_or_die("HTTP/1.0");
986    }
987
988    /*
989     * Rebuild the request line.
990     */
991    freez(*request_line);
992    *request_line = strdup(http->gpc);
993    string_append(request_line, " ");
994
995    if (fwd->forward_host && fwd->type != FORWARD_WEBSERVER)
996    {
997       string_append(request_line, http->url);
998    }
999    else
1000    {
1001       string_append(request_line, http->path);
1002    }
1003    string_append(request_line, " ");
1004    string_append(request_line, http->version);
1005
1006    if (*request_line == NULL)
1007    {
1008       log_error(LOG_LEVEL_FATAL, "Out of memory writing HTTP command");
1009    }
1010    log_error(LOG_LEVEL_HEADER, "New HTTP Request-Line: %s", *request_line);
1011 }
1012
1013
1014 /*********************************************************************
1015  *
1016  * Function    :  change_request_destination
1017  *
1018  * Description :  Parse a (rewritten) request line and regenerate
1019  *                the http request data.
1020  *
1021  * Parameters  :
1022  *          1  :  csp = Current client state (buffers, headers, etc...)
1023  *
1024  * Returns     :  Forwards the parse_http_request() return code.
1025  *                Terminates in case of memory problems.
1026  *
1027  *********************************************************************/
1028 static jb_err change_request_destination(struct client_state *csp)
1029 {
1030    struct http_request *http = csp->http;
1031    jb_err err;
1032
1033    log_error(LOG_LEVEL_REDIRECTS, "Rewrite detected: %s",
1034       csp->headers->first->str);
1035    free_http_request(http);
1036    err = parse_http_request(csp->headers->first->str, http);
1037    if (JB_ERR_OK != err)
1038    {
1039       log_error(LOG_LEVEL_ERROR, "Couldn't parse rewritten request: %s.",
1040          jb_err_to_string(err));
1041    }
1042
1043    return err;
1044 }
1045
1046
1047 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
1048 /*********************************************************************
1049  *
1050  * Function    :  server_response_is_complete
1051  *
1052  * Description :  Determines whether we should stop reading
1053  *                from the server socket.
1054  *
1055  * Parameters  :
1056  *          1  :  csp = Current client state (buffers, headers, etc...)
1057  *          2  :  content_length = Length of content received so far.
1058  *
1059  * Returns     :  TRUE if the response is complete,
1060  *                FALSE otherwise.
1061  *
1062  *********************************************************************/
1063 static int server_response_is_complete(struct client_state *csp,
1064    unsigned long long content_length)
1065 {
1066    int content_length_known = !!(csp->flags & CSP_FLAG_CONTENT_LENGTH_SET);
1067
1068    if (!strcmpic(csp->http->gpc, "HEAD"))
1069    {
1070       /*
1071        * "HEAD" implies no body, we are thus expecting
1072        * no content. XXX: incomplete "list" of methods?
1073        */
1074       csp->expected_content_length = 0;
1075       content_length_known = TRUE;
1076       csp->flags |= CSP_FLAG_SERVER_CONTENT_LENGTH_SET;
1077    }
1078
1079    if (csp->http->status == 204 || csp->http->status == 304)
1080    {
1081       /*
1082        * Expect no body. XXX: incomplete "list" of status codes?
1083        */
1084       csp->expected_content_length = 0;
1085       content_length_known = TRUE;
1086       csp->flags |= CSP_FLAG_SERVER_CONTENT_LENGTH_SET;
1087    }
1088
1089    return (content_length_known && ((0 == csp->expected_content_length)
1090             || (csp->expected_content_length <= content_length)));
1091 }
1092
1093
1094 #ifdef FEATURE_CONNECTION_SHARING
1095 /*********************************************************************
1096  *
1097  * Function    :  wait_for_alive_connections
1098  *
1099  * Description :  Waits for alive connections to timeout.
1100  *
1101  * Parameters  :  N/A
1102  *
1103  * Returns     :  N/A
1104  *
1105  *********************************************************************/
1106 static void wait_for_alive_connections(void)
1107 {
1108    int connections_alive = close_unusable_connections();
1109
1110    while (0 < connections_alive)
1111    {
1112       log_error(LOG_LEVEL_CONNECT,
1113          "Waiting for %d connections to timeout.",
1114          connections_alive);
1115       sleep(60);
1116       connections_alive = close_unusable_connections();
1117    }
1118
1119    log_error(LOG_LEVEL_CONNECT, "No connections to wait for left.");
1120
1121 }
1122 #endif /* def FEATURE_CONNECTION_SHARING */
1123
1124
1125 /*********************************************************************
1126  *
1127  * Function    :  save_connection_destination
1128  *
1129  * Description :  Remembers a connection for reuse later on.
1130  *
1131  * Parameters  :
1132  *          1  :  sfd  = Open socket to remember.
1133  *          2  :  http = The destination for the connection.
1134  *          3  :  fwd  = The forwarder settings used.
1135  *          4  :  server_connection  = storage.
1136  *
1137  * Returns     : void
1138  *
1139  *********************************************************************/
1140 void save_connection_destination(jb_socket sfd,
1141                                  const struct http_request *http,
1142                                  const struct forward_spec *fwd,
1143                                  struct reusable_connection *server_connection)
1144 {
1145    assert(sfd != JB_INVALID_SOCKET);
1146    assert(NULL != http->host);
1147
1148    server_connection->sfd = sfd;
1149    server_connection->host = strdup_or_die(http->host);
1150    server_connection->port = http->port;
1151
1152    assert(NULL != fwd);
1153    assert(server_connection->gateway_host == NULL);
1154    assert(server_connection->gateway_port == 0);
1155    assert(server_connection->forwarder_type == 0);
1156    assert(server_connection->forward_host == NULL);
1157    assert(server_connection->forward_port == 0);
1158
1159    server_connection->forwarder_type = fwd->type;
1160    if (NULL != fwd->gateway_host)
1161    {
1162       server_connection->gateway_host = strdup_or_die(fwd->gateway_host);
1163    }
1164    else
1165    {
1166       server_connection->gateway_host = NULL;
1167    }
1168    server_connection->gateway_port = fwd->gateway_port;
1169    if (NULL != fwd->auth_username)
1170    {
1171       server_connection->auth_username = strdup_or_die(fwd->auth_username);
1172    }
1173    else
1174    {
1175       server_connection->auth_username = NULL;
1176    }
1177    if (NULL != fwd->auth_password)
1178    {
1179       server_connection->auth_password = strdup_or_die(fwd->auth_password);
1180    }
1181    else
1182    {
1183       server_connection->auth_password = NULL;
1184    }
1185
1186    if (NULL != fwd->forward_host)
1187    {
1188       server_connection->forward_host = strdup_or_die(fwd->forward_host);
1189    }
1190    else
1191    {
1192       server_connection->forward_host = NULL;
1193    }
1194    server_connection->forward_port = fwd->forward_port;
1195 }
1196 #endif /* FEATURE_CONNECTION_KEEP_ALIVE */
1197
1198
1199 /*********************************************************************
1200  *
1201  * Function    : verify_request_length
1202  *
1203  * Description : Checks if we already got the whole client requests
1204  *               and sets CSP_FLAG_CLIENT_REQUEST_COMPLETELY_READ if
1205  *               we do.
1206  *
1207  *               Data that doesn't belong to the current request is
1208  *               either thrown away to let the client retry on a clean
1209  *               socket, or stashed to be dealt with after the current
1210  *               request is served.
1211  *
1212  * Parameters  :
1213  *          1  :  csp = Current client state (buffers, headers, etc...)
1214  *
1215  * Returns     :  void
1216  *
1217  *********************************************************************/
1218 static void verify_request_length(struct client_state *csp)
1219 {
1220    unsigned long long buffered_request_bytes =
1221       (unsigned long long)(csp->client_iob->eod - csp->client_iob->cur);
1222
1223    if ((csp->expected_client_content_length != 0)
1224       && (buffered_request_bytes != 0))
1225    {
1226       if (csp->expected_client_content_length >= buffered_request_bytes)
1227       {
1228          csp->expected_client_content_length -= buffered_request_bytes;
1229          log_error(LOG_LEVEL_CONNECT, "Reduced expected bytes to %llu "
1230             "to account for the %llu ones we already got.",
1231             csp->expected_client_content_length, buffered_request_bytes);
1232       }
1233       else
1234       {
1235          assert(csp->client_iob->eod > csp->client_iob->cur + csp->expected_client_content_length);
1236          csp->client_iob->eod = csp->client_iob->cur + csp->expected_client_content_length;
1237          log_error(LOG_LEVEL_CONNECT, "Reducing expected bytes to 0. "
1238             "Marking the server socket tainted after throwing %llu bytes away.",
1239             buffered_request_bytes - csp->expected_client_content_length);
1240          csp->expected_client_content_length = 0;
1241          csp->flags |= CSP_FLAG_SERVER_SOCKET_TAINTED;
1242       }
1243
1244       if (csp->expected_client_content_length == 0)
1245       {
1246          csp->flags |= CSP_FLAG_CLIENT_REQUEST_COMPLETELY_READ;
1247       }
1248    }
1249
1250    if (!(csp->flags & CSP_FLAG_CLIENT_REQUEST_COMPLETELY_READ)
1251       && ((csp->client_iob->cur < csp->client_iob->eod)
1252          || (csp->expected_client_content_length != 0)))
1253    {
1254       if (strcmpic(csp->http->gpc, "GET")
1255          && strcmpic(csp->http->gpc, "HEAD")
1256          && strcmpic(csp->http->gpc, "TRACE")
1257          && strcmpic(csp->http->gpc, "OPTIONS")
1258          && strcmpic(csp->http->gpc, "DELETE"))
1259       {
1260          /* XXX: this is an incomplete hack */
1261          csp->flags &= ~CSP_FLAG_CLIENT_REQUEST_COMPLETELY_READ;
1262          log_error(LOG_LEVEL_CONNECT, "There better be a request body.");
1263       }
1264       else
1265       {
1266          csp->flags |= CSP_FLAG_CLIENT_REQUEST_COMPLETELY_READ;
1267
1268          if ((csp->config->feature_flags & RUNTIME_FEATURE_TOLERATE_PIPELINING) == 0)
1269          {
1270             csp->flags |= CSP_FLAG_SERVER_SOCKET_TAINTED;
1271             log_error(LOG_LEVEL_CONNECT,
1272                "Possible pipeline attempt detected. The connection will not "
1273                "be kept alive and we will only serve the first request.");
1274             /* Nuke the pipelined requests from orbit, just to be sure. */
1275             clear_iob(csp->client_iob);
1276          }
1277          else
1278          {
1279             /*
1280              * Keep the pipelined data around for now, we'll deal with
1281              * it once we're done serving the current request.
1282              */
1283             csp->flags |= CSP_FLAG_PIPELINED_REQUEST_WAITING;
1284             assert(csp->client_iob->eod >= csp->client_iob->cur);
1285             log_error(LOG_LEVEL_CONNECT, "Complete client request followed by "
1286                "%d bytes of pipelined data received.",
1287                (int)(csp->client_iob->eod - csp->client_iob->cur));
1288          }
1289       }
1290    }
1291    else
1292    {
1293       csp->flags |= CSP_FLAG_CLIENT_REQUEST_COMPLETELY_READ;
1294       log_error(LOG_LEVEL_CONNECT, "Complete client request received.");
1295    }
1296 }
1297
1298
1299 /*********************************************************************
1300  *
1301  * Function    :  mark_server_socket_tainted
1302  *
1303  * Description :  Makes sure we don't reuse a server socket
1304  *                (if we didn't read everything the server sent
1305  *                us reusing the socket would lead to garbage).
1306  *
1307  * Parameters  :
1308  *          1  :  csp = Current client state (buffers, headers, etc...)
1309  *
1310  * Returns     :  void.
1311  *
1312  *********************************************************************/
1313 static void mark_server_socket_tainted(struct client_state *csp)
1314 {
1315    /*
1316     * For consistency we always mark the server socket
1317     * tainted, however, to reduce the log noise we only
1318     * emit a log message if the server socket could have
1319     * actually been reused.
1320     */
1321    if ((csp->flags & CSP_FLAG_SERVER_CONNECTION_KEEP_ALIVE)
1322       && !(csp->flags & CSP_FLAG_SERVER_SOCKET_TAINTED))
1323    {
1324       log_error(LOG_LEVEL_CONNECT,
1325          "Marking the server socket %d tainted.",
1326          csp->server_connection.sfd);
1327    }
1328    csp->flags |= CSP_FLAG_SERVER_SOCKET_TAINTED;
1329 }
1330
1331 /*********************************************************************
1332  *
1333  * Function    :  get_request_line
1334  *
1335  * Description : Read the client request line.
1336  *
1337  * Parameters  :
1338  *          1  :  csp = Current client state (buffers, headers, etc...)
1339  *
1340  * Returns     :  Pointer to request line or NULL in case of errors.
1341  *
1342  *********************************************************************/
1343 static char *get_request_line(struct client_state *csp)
1344 {
1345    char buf[BUFFER_SIZE];
1346    char *request_line = NULL;
1347    int len;
1348
1349    memset(buf, 0, sizeof(buf));
1350
1351    if ((csp->flags & CSP_FLAG_PIPELINED_REQUEST_WAITING) != 0)
1352    {
1353       /*
1354        * If there are multiple pipelined requests waiting,
1355        * the flag will be set again once the next request
1356        * has been parsed.
1357        */
1358       csp->flags &= ~CSP_FLAG_PIPELINED_REQUEST_WAITING;
1359
1360       request_line = get_header(csp->client_iob);
1361       if ((NULL != request_line) && ('\0' != *request_line))
1362       {
1363          return request_line;
1364       }
1365       else
1366       {
1367          log_error(LOG_LEVEL_CONNECT, "No complete request line "
1368             "received yet. Continuing reading from %d.", csp->cfd);
1369       }
1370    }
1371
1372    do
1373    {
1374       if (
1375 #ifdef FUZZ
1376           0 == (csp->flags & CSP_FLAG_FUZZED_INPUT) &&
1377 #endif
1378           !data_is_available(csp->cfd, csp->config->socket_timeout)
1379           )
1380       {
1381          if (socket_is_still_alive(csp->cfd))
1382          {
1383             log_error(LOG_LEVEL_CONNECT,
1384                "No request line on socket %d received in time. Timeout: %d.",
1385                csp->cfd, csp->config->socket_timeout);
1386             write_socket_delayed(csp->cfd, CLIENT_CONNECTION_TIMEOUT_RESPONSE,
1387                strlen(CLIENT_CONNECTION_TIMEOUT_RESPONSE),
1388                get_write_delay(csp));
1389          }
1390          else
1391          {
1392             log_error(LOG_LEVEL_CONNECT,
1393                "The client side of the connection on socket %d got "
1394                "closed without sending a complete request line.", csp->cfd);
1395          }
1396          return NULL;
1397       }
1398
1399       len = read_socket(csp->cfd, buf, sizeof(buf) - 1);
1400
1401       if (len <= 0) return NULL;
1402
1403       /*
1404        * If there is no memory left for buffering the
1405        * request, there is nothing we can do but hang up
1406        */
1407       if (add_to_iob(csp->client_iob, csp->config->buffer_limit, buf, len))
1408       {
1409          return NULL;
1410       }
1411
1412       request_line = get_header(csp->client_iob);
1413
1414    } while ((NULL != request_line) && ('\0' == *request_line));
1415
1416    return request_line;
1417
1418 }
1419
1420 enum chunk_status
1421 {
1422    CHUNK_STATUS_MISSING_DATA,
1423    CHUNK_STATUS_BODY_COMPLETE,
1424    CHUNK_STATUS_PARSE_ERROR
1425 };
1426
1427
1428 /*********************************************************************
1429  *
1430  * Function    :  chunked_body_is_complete
1431  *
1432  * Description :  Figures out whether or not a chunked body is complete.
1433  *
1434  *                Currently it always starts at the beginning of the
1435  *                buffer which is somewhat wasteful and prevents Privoxy
1436  *                from starting to forward the correctly parsed chunks
1437  *                as soon as theoretically possible.
1438  *
1439  *                Should be modified to work with a common buffer,
1440  *                and allow the caller to skip already parsed chunks.
1441  *
1442  *                This would allow the function to be used for unbuffered
1443  *                response bodies as well.
1444  *
1445  * Parameters  :
1446  *          1  :  iob = Buffer with the body to check.
1447  *          2  :  length = Length of complete body
1448  *
1449  * Returns     :  Enum with the result of the check.
1450  *
1451  *********************************************************************/
1452 static enum chunk_status chunked_body_is_complete(struct iob *iob, size_t *length)
1453 {
1454    unsigned int chunksize;
1455    char *p = iob->cur;
1456
1457    do
1458    {
1459       /*
1460        * We need at least a single digit, followed by "\r\n",
1461        * followed by an unknown amount of data, followed by "\r\n".
1462        */
1463       if (p + 5 > iob->eod)
1464       {
1465          return CHUNK_STATUS_MISSING_DATA;
1466       }
1467       if (sscanf(p, "%x", &chunksize) != 1)
1468       {
1469          return CHUNK_STATUS_PARSE_ERROR;
1470       }
1471
1472       /*
1473        * We want at least a single digit, followed by "\r\n",
1474        * followed by the specified amount of data, followed by "\r\n".
1475        */
1476       if (p + chunksize + 5 > iob->eod)
1477       {
1478          return CHUNK_STATUS_MISSING_DATA;
1479       }
1480
1481       /* Skip chunk-size. */
1482       p = strstr(p, "\r\n");
1483       if (NULL == p)
1484       {
1485          return CHUNK_STATUS_PARSE_ERROR;
1486       }
1487       /* Move beyond the chunkdata. */
1488       p += 2 + chunksize;
1489
1490       /* There should be another "\r\n" to skip */
1491       if (memcmp(p, "\r\n", 2))
1492       {
1493          return CHUNK_STATUS_PARSE_ERROR;
1494       }
1495       p += 2;
1496    } while (chunksize > 0U);
1497
1498    *length = (size_t)(p - iob->cur);
1499    assert(*length <= (size_t)(iob->eod - iob->cur));
1500    assert(p <= iob->eod);
1501
1502    return CHUNK_STATUS_BODY_COMPLETE;
1503
1504 }
1505
1506
1507 /*********************************************************************
1508  *
1509  * Function    : receive_chunked_client_request_body
1510  *
1511  * Description : Read the chunk-encoded client request body.
1512  *               Failures are dealt with.
1513  *
1514  * Parameters  :
1515  *          1  :  csp = Current client state (buffers, headers, etc...)
1516  *
1517  * Returns     :  JB_ERR_OK or JB_ERR_PARSE
1518  *
1519  *********************************************************************/
1520 static jb_err receive_chunked_client_request_body(struct client_state *csp)
1521 {
1522    size_t body_length;
1523    enum chunk_status status;
1524
1525    while (CHUNK_STATUS_MISSING_DATA ==
1526       (status = chunked_body_is_complete(csp->client_iob, &body_length)))
1527    {
1528       char buf[BUFFER_SIZE];
1529       int len;
1530
1531       if (!data_is_available(csp->cfd, csp->config->socket_timeout))
1532       {
1533          log_error(LOG_LEVEL_ERROR,
1534             "Timeout while waiting for the client body.");
1535          break;
1536       }
1537       len = read_socket(csp->cfd, buf, sizeof(buf) - 1);
1538       if (len <= 0)
1539       {
1540          log_error(LOG_LEVEL_ERROR, "Read the client body failed: %E");
1541          break;
1542       }
1543       if (add_to_iob(csp->client_iob, csp->config->buffer_limit, buf, len))
1544       {
1545          break;
1546       }
1547    }
1548    if (status != CHUNK_STATUS_BODY_COMPLETE)
1549    {
1550       write_socket_delayed(csp->cfd, CLIENT_BODY_PARSE_ERROR_RESPONSE,
1551          strlen(CLIENT_BODY_PARSE_ERROR_RESPONSE), get_write_delay(csp));
1552       log_error(LOG_LEVEL_CLF,
1553          "%s - - [%T] \"Failed reading chunked client body\" 400 0", csp->ip_addr_str);
1554       return JB_ERR_PARSE;
1555    }
1556    log_error(LOG_LEVEL_CONNECT,
1557       "Chunked client body completely read. Length: %lu", body_length);
1558    csp->expected_client_content_length = body_length;
1559
1560    return JB_ERR_OK;
1561
1562 }
1563
1564
1565 #ifdef FUZZ
1566 /*********************************************************************
1567  *
1568  * Function    :  fuzz_chunked_transfer_encoding
1569  *
1570  * Description :  Treat the fuzzed input as chunked transfer encoding
1571  *                to check and dechunk.
1572  *
1573  * Parameters  :
1574  *          1  :  csp      = Used to store the data.
1575  *          2  :  fuzz_input_file = File to read the input from.
1576  *
1577  * Returns     : Result of dechunking
1578  *
1579  *********************************************************************/
1580 extern int fuzz_chunked_transfer_encoding(struct client_state *csp, char *fuzz_input_file)
1581 {
1582    size_t length;
1583    size_t size = (size_t)(csp->iob->eod - csp->iob->cur);
1584    enum chunk_status status;
1585
1586    status = chunked_body_is_complete(csp->iob, &length);
1587    if (CHUNK_STATUS_BODY_COMPLETE != status)
1588    {
1589       log_error(LOG_LEVEL_INFO, "Chunked body is incomplete or invalid");
1590    }
1591
1592    return (JB_ERR_OK == remove_chunked_transfer_coding(csp->iob->cur, &size));
1593
1594 }
1595
1596
1597 /*********************************************************************
1598  *
1599  * Function    : fuzz_client_request
1600  *
1601  * Description : Try to get a client request from the fuzzed input.
1602  *
1603  * Parameters  :
1604  *          1  :  csp = Current client state (buffers, headers, etc...)
1605  *          2  :  fuzz_input_file = File to read the input from.
1606  *
1607  * Returns     :  Result of fuzzing.
1608  *
1609  *********************************************************************/
1610 extern int fuzz_client_request(struct client_state *csp, char *fuzz_input_file)
1611 {
1612    jb_err err;
1613
1614    csp->cfd = 0;
1615    csp->ip_addr_str = "fuzzer";
1616
1617    if (strcmp(fuzz_input_file, "-") != 0)
1618    {
1619       log_error(LOG_LEVEL_FATAL,
1620          "Fuzzed client requests can currently only be read from stdin (-).");
1621    }
1622    err = receive_client_request(csp);
1623    if (err != JB_ERR_OK)
1624    {
1625       return 1;
1626    }
1627    err = parse_client_request(csp);
1628    if (err != JB_ERR_OK)
1629    {
1630       return 1;
1631    }
1632
1633    return 0;
1634
1635 }
1636 #endif  /* def FUZZ */
1637
1638
1639 #ifdef FEATURE_FORCE_LOAD
1640 /*********************************************************************
1641  *
1642  * Function    :  force_required
1643  *
1644  * Description : Checks a request line to see if it contains
1645  *               the FORCE_PREFIX. If it does, it is removed
1646  *               unless enforcing requests has beend disabled.
1647  *
1648  * Parameters  :
1649  *          1  :  request_line = HTTP request line
1650  *
1651  * Returns     :  TRUE if force is required, FALSE otherwise.
1652  *
1653  *********************************************************************/
1654 static int force_required(const struct client_state *csp, char *request_line)
1655 {
1656    char *p;
1657
1658    p = strstr(request_line, "http://");
1659    if (p != NULL)
1660    {
1661       /* Skip protocol */
1662       p += strlen("http://");
1663    }
1664    else
1665    {
1666       /* Intercepted request usually don't specify the protocol. */
1667       p = request_line;
1668    }
1669
1670    /* Go to the beginning of the path */
1671    p = strstr(p, "/");
1672    if (p == NULL)
1673    {
1674       /*
1675        * If the path is missing the request line is invalid and we
1676        * are done here. The client-visible rejection happens later on.
1677        */
1678       return 0;
1679    }
1680
1681    if (0 == strncmpic(p, FORCE_PREFIX, strlen(FORCE_PREFIX) - 1))
1682    {
1683       if (!(csp->config->feature_flags & RUNTIME_FEATURE_ENFORCE_BLOCKS))
1684       {
1685          /* XXX: Should clean more carefully */
1686          strclean(request_line, FORCE_PREFIX);
1687          log_error(LOG_LEVEL_FORCE,
1688             "Enforcing request: \"%s\".", request_line);
1689
1690          return 1;
1691       }
1692       log_error(LOG_LEVEL_FORCE,
1693          "Ignored force prefix in request: \"%s\".", request_line);
1694    }
1695
1696    return 0;
1697
1698 }
1699 #endif /* def FEATURE_FORCE_LOAD */
1700
1701
1702 /*********************************************************************
1703  *
1704  * Function    :  receive_client_request
1705  *
1706  * Description : Read the client's request (more precisely the
1707  *               client headers) and answer it if necessary.
1708  *
1709  * Parameters  :
1710  *          1  :  csp = Current client state (buffers, headers, etc...)
1711  *
1712  * Returns     :  JB_ERR_OK, JB_ERR_PARSE or JB_ERR_MEMORY
1713  *
1714  *********************************************************************/
1715 static jb_err receive_client_request(struct client_state *csp)
1716 {
1717    char buf[BUFFER_SIZE];
1718    char *p;
1719    char *req = NULL;
1720    struct http_request *http;
1721    int len;
1722    jb_err err;
1723
1724    /* Temporary copy of the client's headers before they get enlisted in csp->headers */
1725    struct list header_list;
1726    struct list *headers = &header_list;
1727
1728    /* We don't care if the arriving data is a valid HTTP request or not. */
1729    csp->requests_received_total++;
1730
1731    http = csp->http;
1732
1733    memset(buf, 0, sizeof(buf));
1734
1735    req = get_request_line(csp);
1736    if (req == NULL)
1737    {
1738       mark_server_socket_tainted(csp);
1739       return JB_ERR_PARSE;
1740    }
1741    assert(*req != '\0');
1742
1743    if (client_protocol_is_unsupported(csp, req))
1744    {
1745       return JB_ERR_PARSE;
1746    }
1747
1748 #ifdef FEATURE_FORCE_LOAD
1749    if (force_required(csp, req))
1750    {
1751       csp->flags |= CSP_FLAG_FORCED;
1752    }
1753 #endif /* def FEATURE_FORCE_LOAD */
1754
1755    err = parse_http_request(req, http);
1756    freez(req);
1757    if (JB_ERR_OK != err)
1758    {
1759       write_socket_delayed(csp->cfd, CHEADER, strlen(CHEADER),
1760          get_write_delay(csp));
1761       /* XXX: Use correct size */
1762       log_error(LOG_LEVEL_CLF, "%s - - [%T] \"Invalid request\" 400 0", csp->ip_addr_str);
1763       log_error(LOG_LEVEL_ERROR,
1764          "Couldn't parse request line received from %s: %s",
1765          csp->ip_addr_str, jb_err_to_string(err));
1766
1767       free_http_request(http);
1768       return JB_ERR_PARSE;
1769    }
1770
1771    /* grab the rest of the client's headers */
1772    init_list(headers);
1773    for (;;)
1774    {
1775       p = get_header(csp->client_iob);
1776
1777       if (p == NULL)
1778       {
1779          /* There are no additional headers to read. */
1780          break;
1781       }
1782
1783       if (*p == '\0')
1784       {
1785          /*
1786           * We didn't receive a complete header
1787           * line yet, get the rest of it.
1788           */
1789          if (!data_is_available(csp->cfd, csp->config->socket_timeout))
1790          {
1791             log_error(LOG_LEVEL_ERROR,
1792                "Stopped grabbing the client headers.");
1793             destroy_list(headers);
1794             return JB_ERR_PARSE;
1795          }
1796
1797          len = read_socket(csp->cfd, buf, sizeof(buf) - 1);
1798          if (len <= 0)
1799          {
1800             log_error(LOG_LEVEL_ERROR, "read from client failed: %E");
1801             destroy_list(headers);
1802             return JB_ERR_PARSE;
1803          }
1804
1805          if (add_to_iob(csp->client_iob, csp->config->buffer_limit, buf, len))
1806          {
1807             /*
1808              * If there is no memory left for buffering the
1809              * request, there is nothing we can do but hang up
1810              */
1811             destroy_list(headers);
1812             return JB_ERR_MEMORY;
1813          }
1814       }
1815       else
1816       {
1817          if (!strncmpic(p, "Transfer-Encoding:", 18))
1818          {
1819             /*
1820              * XXX: should be called through sed()
1821              *      but currently can't.
1822              */
1823             client_transfer_encoding(csp, &p);
1824          }
1825          /*
1826           * We were able to read a complete
1827           * header and can finally enlist it.
1828           */
1829          enlist(headers, p);
1830          freez(p);
1831       }
1832    }
1833
1834    if (http->host == NULL)
1835    {
1836       /*
1837        * If we still don't know the request destination,
1838        * the request is invalid or the client uses
1839        * Privoxy without its knowledge.
1840        */
1841       if (JB_ERR_OK != get_request_destination_elsewhere(csp, headers))
1842       {
1843          /*
1844           * Our attempts to get the request destination
1845           * elsewhere failed or Privoxy is configured
1846           * to only accept proxy requests.
1847           *
1848           * An error response has already been send
1849           * and we're done here.
1850           */
1851          return JB_ERR_PARSE;
1852       }
1853    }
1854
1855 #ifdef FEATURE_CLIENT_TAGS
1856    /* XXX: If the headers were enlisted sooner, passing csp would do. */
1857    set_client_address(csp, headers);
1858    get_tag_list_for_client(csp->client_tags, csp->client_address);
1859 #endif
1860
1861    /*
1862     * Determine the actions for this URL
1863     */
1864 #ifdef FEATURE_TOGGLE
1865    if (!(csp->flags & CSP_FLAG_TOGGLED_ON))
1866    {
1867       /* Most compatible set of actions (i.e. none) */
1868       init_current_action(csp->action);
1869    }
1870    else
1871 #endif /* ndef FEATURE_TOGGLE */
1872    {
1873       get_url_actions(csp, http);
1874    }
1875
1876    enlist(csp->headers, http->cmd);
1877
1878    /* Append the previously read headers */
1879    err = list_append_list_unique(csp->headers, headers);
1880    destroy_list(headers);
1881
1882    return err;
1883
1884 }
1885
1886
1887 /*********************************************************************
1888  *
1889  * Function    : parse_client_request
1890  *
1891  * Description : Parses the client's request and decides what to do
1892  *               with it.
1893  *
1894  *               Note that since we're not using select() we could get
1895  *               blocked here if a client connected, then didn't say
1896  *               anything!
1897  *
1898  * Parameters  :
1899  *          1  :  csp = Current client state (buffers, headers, etc...)
1900  *
1901  * Returns     :  JB_ERR_OK or JB_ERR_PARSE
1902  *
1903  *********************************************************************/
1904 static jb_err parse_client_request(struct client_state *csp)
1905 {
1906    struct http_request *http = csp->http;
1907    jb_err err;
1908
1909 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
1910    if ((csp->config->feature_flags & RUNTIME_FEATURE_CONNECTION_KEEP_ALIVE)
1911     && (!strcmpic(csp->http->version, "HTTP/1.1"))
1912     && (csp->http->ssl == 0))
1913    {
1914       /* Assume persistence until further notice */
1915       csp->flags |= CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE;
1916    }
1917 #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */
1918
1919    if (csp->http->ssl == 0)
1920    {
1921       /*
1922        * This whole block belongs to chat() but currently
1923        * has to be executed before sed().
1924        */
1925       if (csp->flags & CSP_FLAG_CHUNKED_CLIENT_BODY)
1926       {
1927          if (receive_chunked_client_request_body(csp) != JB_ERR_OK)
1928          {
1929             return JB_ERR_PARSE;
1930          }
1931       }
1932       else
1933       {
1934          csp->expected_client_content_length = get_expected_content_length(csp->headers);
1935       }
1936       verify_request_length(csp);
1937    }
1938 #ifndef FEATURE_HTTPS_INSPECTION
1939    else
1940    {
1941       csp->flags |= CSP_FLAG_SERVER_SOCKET_TAINTED;
1942    }
1943 #endif
1944
1945    err = sed(csp, FILTER_CLIENT_HEADERS);
1946    if (JB_ERR_OK != err)
1947    {
1948       log_error(LOG_LEVEL_ERROR, "Failed to parse client request from %s.",
1949          csp->ip_addr_str);
1950       log_error(LOG_LEVEL_CLF, "%s - - [%T] \"%s\" 400 0",
1951          csp->ip_addr_str, csp->http->cmd);
1952       write_socket_delayed(csp->cfd, CHEADER, strlen(CHEADER), get_write_delay(csp));
1953       return JB_ERR_PARSE;
1954    }
1955    csp->flags |= CSP_FLAG_CLIENT_HEADER_PARSING_DONE;
1956
1957    /* Check request line for rewrites. */
1958    if ((NULL == csp->headers->first->str)
1959       || (strcmp(http->cmd, csp->headers->first->str) &&
1960          (JB_ERR_OK != change_request_destination(csp))))
1961    {
1962       /*
1963        * A header filter broke the request line - bail out.
1964        */
1965       write_socket_delayed(csp->cfd, MESSED_UP_REQUEST_RESPONSE,
1966          strlen(MESSED_UP_REQUEST_RESPONSE), get_write_delay(csp));
1967       /* XXX: Use correct size */
1968       log_error(LOG_LEVEL_CLF,
1969          "%s - - [%T] \"Invalid request generated\" 500 0", csp->ip_addr_str);
1970       log_error(LOG_LEVEL_ERROR,
1971          "Invalid request line after applying header filters.");
1972       free_http_request(http);
1973
1974       return JB_ERR_PARSE;
1975    }
1976
1977    if (client_has_unsupported_expectations(csp))
1978    {
1979       return JB_ERR_PARSE;
1980    }
1981
1982    return JB_ERR_OK;
1983
1984 }
1985
1986
1987 /*********************************************************************
1988  *
1989  * Function    : send_http_request
1990  *
1991  * Description : Sends the HTTP headers from the client request
1992  *               and all the body data that has already been received.
1993  *
1994  * Parameters  :
1995  *          1  :  csp = Current client state (buffers, headers, etc...)
1996  *
1997  * Returns     :  0 on success, anything else is an error.
1998  *
1999  *********************************************************************/
2000 static int send_http_request(struct client_state *csp)
2001 {
2002    char *hdr;
2003    int write_failure;
2004
2005    hdr = list_to_text(csp->headers);
2006    if (hdr == NULL)
2007    {
2008       /* FIXME Should handle error properly */
2009       log_error(LOG_LEVEL_FATAL, "Out of memory parsing client header");
2010    }
2011    list_remove_all(csp->headers);
2012
2013    /*
2014     * Write the client's (modified) header to the server
2015     * (along with anything else that may be in the buffer)
2016     */
2017    write_failure = 0 != write_socket(csp->server_connection.sfd, hdr, strlen(hdr));
2018    freez(hdr);
2019
2020    if (write_failure)
2021    {
2022       log_error(LOG_LEVEL_CONNECT, "Failed sending request headers to: %s: %E",
2023          csp->http->hostport);
2024    }
2025    else if (((csp->flags & CSP_FLAG_PIPELINED_REQUEST_WAITING) == 0)
2026       && (flush_iob(csp->server_connection.sfd, csp->client_iob, 0) < 0))
2027    {
2028       write_failure = 1;
2029       log_error(LOG_LEVEL_CONNECT, "Failed sending request body to: %s: %E",
2030          csp->http->hostport);
2031    }
2032
2033    return write_failure;
2034
2035 }
2036
2037
2038 #ifdef FEATURE_HTTPS_INSPECTION
2039 /*********************************************************************
2040  *
2041  * Function    : receive_and_send_encrypted_post_data
2042  *
2043  * Description : Reads remaining POST data from the client and sends
2044  *               it to the server.
2045  *
2046  * Parameters  :
2047  *          1  :  csp = Current client state (buffers, headers, etc...)
2048  *
2049  * Returns     :  0 on success, anything else is an error.
2050  *
2051  *********************************************************************/
2052 static int receive_and_send_encrypted_post_data(struct client_state *csp)
2053 {
2054    int content_length_known = csp->expected_client_content_length != 0;
2055
2056    while (is_ssl_pending(&(csp->ssl_client_attr))
2057       || (content_length_known && csp->expected_client_content_length != 0))
2058    {
2059       unsigned char buf[BUFFER_SIZE];
2060       int len;
2061       int max_bytes_to_read = sizeof(buf);
2062
2063       if (content_length_known && csp->expected_client_content_length < sizeof(buf))
2064       {
2065          max_bytes_to_read = (int)csp->expected_client_content_length;
2066       }
2067       log_error(LOG_LEVEL_CONNECT,
2068          "Waiting for up to %d bytes of POST data from the client.",
2069          max_bytes_to_read);
2070       len = ssl_recv_data(&(csp->ssl_client_attr), buf,
2071          (unsigned)max_bytes_to_read);
2072       if (len == -1)
2073       {
2074          return 1;
2075       }
2076       if (len == 0)
2077       {
2078          /* XXX: Does this actually happen? */
2079          break;
2080       }
2081       log_error(LOG_LEVEL_CONNECT, "Forwarding %d bytes of encrypted POST data",
2082          len);
2083       len = ssl_send_data(&(csp->ssl_server_attr), buf, (size_t)len);
2084       if (len == -1)
2085       {
2086          return 1;
2087       }
2088       if (csp->expected_client_content_length != 0)
2089       {
2090          if (csp->expected_client_content_length >= len)
2091          {
2092             csp->expected_client_content_length -= (unsigned)len;
2093          }
2094          if (csp->expected_client_content_length == 0)
2095          {
2096             log_error(LOG_LEVEL_CONNECT, "Forwarded the last %d bytes", len);
2097             break;
2098          }
2099       }
2100    }
2101
2102    log_error(LOG_LEVEL_CONNECT, "Done forwarding encrypted POST data");
2103
2104    return 0;
2105
2106 }
2107
2108
2109 /*********************************************************************
2110  *
2111  * Function    : send_https_request
2112  *
2113  * Description : Sends the HTTP headers from the client request
2114  *               and all the body data that has already been received.
2115  *
2116  * Parameters  :
2117  *          1  :  csp = Current client state (buffers, headers, etc...)
2118  *
2119  * Returns     :  0 on success, anything else is an error.
2120  *
2121  *********************************************************************/
2122 static int send_https_request(struct client_state *csp)
2123 {
2124    char *hdr;
2125    int ret;
2126    long flushed = 0;
2127
2128    hdr = list_to_text(csp->https_headers);
2129    if (hdr == NULL)
2130    {
2131       /* FIXME Should handle error properly */
2132       log_error(LOG_LEVEL_FATAL, "Out of memory parsing client header");
2133    }
2134    list_remove_all(csp->https_headers);
2135
2136    /*
2137     * Write the client's (modified) header to the server
2138     * (along with anything else that may be in the buffer)
2139     */
2140    ret = ssl_send_data(&(csp->ssl_server_attr),
2141       (const unsigned char *)hdr, strlen(hdr));
2142    freez(hdr);
2143
2144    if (ret < 0)
2145    {
2146       log_error(LOG_LEVEL_CONNECT,
2147          "Failed sending encrypted request headers to: %s: %E",
2148          csp->http->hostport);
2149       mark_server_socket_tainted(csp);
2150       return 1;
2151    }
2152
2153    if (((csp->flags & CSP_FLAG_PIPELINED_REQUEST_WAITING) == 0)
2154       && ((flushed = ssl_flush_socket(&(csp->ssl_server_attr),
2155             csp->client_iob)) < 0))
2156    {
2157       log_error(LOG_LEVEL_CONNECT, "Failed sending request body to: %s: %E",
2158          csp->http->hostport);
2159       return 1;
2160    }
2161    if (flushed != 0 || csp->expected_client_content_length != 0)
2162    {
2163       if (csp->expected_client_content_length != 0)
2164       {
2165          if (csp->expected_client_content_length < flushed)
2166          {
2167             log_error(LOG_LEVEL_ERROR,
2168                "Flushed %ld bytes of request body while only expecting %llu",
2169                flushed, csp->expected_client_content_length);
2170             csp->expected_client_content_length = 0;
2171          }
2172          else
2173          {
2174             log_error(LOG_LEVEL_CONNECT,
2175                "Flushed %ld bytes of request body while expecting %llu",
2176                flushed, csp->expected_client_content_length);
2177             csp->expected_client_content_length -= (unsigned)flushed;
2178             if (receive_and_send_encrypted_post_data(csp))
2179             {
2180                return 1;
2181             }
2182          }
2183       }
2184       else
2185       {
2186          log_error(LOG_LEVEL_CONNECT,
2187             "Flushed %ld bytes of request body", flushed);
2188       }
2189    }
2190
2191    log_error(LOG_LEVEL_CONNECT, "Encrypted request sent");
2192
2193    return 0;
2194
2195 }
2196
2197
2198 /*********************************************************************
2199  *
2200  * Function    :  receive_encrypted_request
2201  *
2202  * Description :  Receives an encrypted request.
2203  *
2204  * Parameters  :
2205  *          1  :  csp = Current client state (buffers, headers, etc...)
2206  *
2207  * Returns     :  JB_ERR_OK on success,
2208  *                JB_ERR_PARSE or JB_ERR_MEMORY otherwise
2209  *
2210  *********************************************************************/
2211 static jb_err receive_encrypted_request(struct client_state *csp)
2212 {
2213    char buf[BUFFER_SIZE];
2214    int len;
2215    char *p;
2216
2217    do
2218    {
2219       log_error(LOG_LEVEL_HEADER, "Reading encrypted headers");
2220       if (!is_ssl_pending(&(csp->ssl_client_attr)) &&
2221           !data_is_available(csp->cfd, csp->config->socket_timeout))
2222       {
2223          log_error(LOG_LEVEL_CONNECT,
2224             "Socket %d timed out while waiting for client headers", csp->cfd);
2225          return JB_ERR_PARSE;
2226       }
2227       len = ssl_recv_data(&(csp->ssl_client_attr),
2228          (unsigned char *)buf, sizeof(buf));
2229       if (len == 0)
2230       {
2231          log_error(LOG_LEVEL_CONNECT,
2232             "Socket %d closed while waiting for client headers", csp->cfd);
2233          return JB_ERR_PARSE;
2234       }
2235       if (len == -1)
2236       {
2237          return JB_ERR_PARSE;
2238       }
2239       if (add_to_iob(csp->client_iob, csp->config->buffer_limit, buf, len))
2240       {
2241          return JB_ERR_MEMORY;
2242       }
2243       p = strstr(csp->client_iob->cur, "\r\n\r\n");
2244    } while (p == NULL);
2245
2246    log_error(LOG_LEVEL_HEADER, "Encrypted headers received completely");
2247
2248    return JB_ERR_OK;
2249 }
2250
2251
2252 /*********************************************************************
2253  *
2254  * Function    :  process_encrypted_request
2255  *
2256  * Description :  Receives and parses an encrypted request.
2257  *
2258  * Parameters  :
2259  *          1  :  csp = Current client state (buffers, headers, etc...)
2260  *
2261  * Returns     :  JB_ERR_OK on success,
2262  *                JB_ERR_PARSE or JB_ERR_MEMORY otherwise
2263  *
2264  *********************************************************************/
2265 static jb_err process_encrypted_request(struct client_state *csp)
2266 {
2267    char *p;
2268    char *request_line;
2269    jb_err err;
2270    /* Temporary copy of the client's headers before they get enlisted in csp->https_headers */
2271    struct list header_list;
2272    struct list *headers = &header_list;
2273
2274    assert(csp->ssl_with_client_is_opened);
2275
2276 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
2277    if (csp->config->feature_flags & RUNTIME_FEATURE_CONNECTION_KEEP_ALIVE)
2278    {
2279       csp->flags |= CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE;
2280    }
2281 #endif
2282    err = receive_encrypted_request(csp);
2283    if (err != JB_ERR_OK)
2284    {
2285       if (csp->client_iob->cur == NULL ||
2286           csp->client_iob->cur == csp->client_iob->eod)
2287       {
2288          /*
2289           * We did not receive any data, most likely because the
2290           * client is done. Don't log this as a parse failure.
2291           */
2292          return JB_ERR_PARSE;
2293       }
2294       /* XXX: Also used for JB_ERR_MEMORY */
2295       log_error(LOG_LEVEL_ERROR, "Failed to receive encrypted request: %s",
2296          jb_err_to_string(err));
2297       ssl_send_data_delayed(&(csp->ssl_client_attr),
2298          (const unsigned char *)CHEADER, strlen(CHEADER), get_write_delay(csp));
2299       return err;
2300    }
2301
2302    /* We don't need get_request_line() because the whole HTTP head is buffered. */
2303    request_line = get_header(csp->client_iob);
2304    if (request_line == NULL)
2305    {
2306       log_error(LOG_LEVEL_ERROR, "Failed to get the encrypted request line");
2307       ssl_send_data_delayed(&(csp->ssl_client_attr),
2308          (const unsigned char *)CHEADER, strlen(CHEADER), get_write_delay(csp));
2309       return JB_ERR_PARSE;
2310    }
2311    assert(*request_line != '\0');
2312
2313    if (client_protocol_is_unsupported(csp, request_line))
2314    {
2315       /*
2316        * If the protocol is unsupported we're done here.
2317        * client_protocol_is_unsupported() took care of sending
2318        * the error response and logging the error message.
2319        */
2320       return JB_ERR_PARSE;
2321    }
2322
2323 #ifdef FEATURE_FORCE_LOAD
2324    if (force_required(csp, request_line))
2325    {
2326       csp->flags |= CSP_FLAG_FORCED;
2327    }
2328 #endif /* def FEATURE_FORCE_LOAD */
2329
2330    free_http_request(csp->http);
2331
2332    err = parse_http_request(request_line, csp->http);
2333    /* XXX: Restore ssl setting. This is ugly */
2334    csp->http->client_ssl = 1;
2335    csp->http->server_ssl = 1;
2336
2337    freez(request_line);
2338    if (JB_ERR_OK != err)
2339    {
2340       ssl_send_data_delayed(&(csp->ssl_client_attr),
2341          (const unsigned char *)CHEADER, strlen(CHEADER), get_write_delay(csp));
2342       /* XXX: Use correct size */
2343       log_error(LOG_LEVEL_CLF, "%s - - [%T] \"Invalid request\" 400 0", csp->ip_addr_str);
2344       log_error(LOG_LEVEL_ERROR,
2345          "Couldn't parse request line received from %s: %s",
2346          csp->ip_addr_str, jb_err_to_string(err));
2347
2348       free_http_request(csp->http);
2349       return JB_ERR_PARSE;
2350    }
2351
2352    /* Parse the rest of the client's headers. */
2353    init_list(headers);
2354    for (;;)
2355    {
2356       p = get_header(csp->client_iob);
2357
2358       if (p == NULL)
2359       {
2360          /* There are no additional headers to read. */
2361          break;
2362       }
2363       enlist(headers, p);
2364       freez(p);
2365    }
2366
2367    if (JB_ERR_OK != get_destination_from_https_headers(headers, csp->http))
2368    {
2369       /*
2370        * Our attempts to get the request destination
2371        * elsewhere failed.
2372        */
2373       log_error(LOG_LEVEL_ERROR,
2374          "Failed to get the encrypted request destination");
2375       ssl_send_data_delayed(&(csp->ssl_client_attr),
2376          (const unsigned char *)CHEADER, strlen(CHEADER), get_write_delay(csp));
2377       return JB_ERR_PARSE;
2378    }
2379
2380    /* Split the domain we just got for pattern matching */
2381    init_domain_components(csp->http);
2382
2383 #ifdef FEATURE_CLIENT_TAGS
2384    /* XXX: If the headers were enlisted sooner, passing csp would do. */
2385    if (csp->client_address == NULL)
2386    {
2387       set_client_address(csp, headers);
2388       get_tag_list_for_client(csp->client_tags, csp->client_address);
2389    }
2390 #endif
2391
2392 #ifdef FEATURE_TOGGLE
2393    if ((csp->flags & CSP_FLAG_TOGGLED_ON) != 0)
2394 #endif
2395    {
2396       /*
2397        * Determine the actions for this request after
2398        * clearing the ones from the previous one.
2399        */
2400       free_current_action(csp->action);
2401       get_url_actions(csp, csp->http);
2402    }
2403
2404    enlist(csp->https_headers, csp->http->cmd);
2405
2406    /* Append the previously read headers */
2407    err = list_append_list_unique(csp->https_headers, headers);
2408    destroy_list(headers);
2409    if (JB_ERR_OK != err)
2410    {
2411       /* XXX: Send error message */
2412       return err;
2413    }
2414
2415    /* XXX: Work around crash */
2416    csp->error_message = NULL;
2417
2418    /* XXX: Why do this here? */
2419    csp->http->ssl = 1;
2420
2421    err = sed_https(csp);
2422    if (JB_ERR_OK != err)
2423    {
2424       ssl_send_data_delayed(&(csp->ssl_client_attr),
2425          (const unsigned char *)CHEADER, strlen(CHEADER), get_write_delay(csp));
2426       log_error(LOG_LEVEL_ERROR, "Failed to parse client request from %s.",
2427          csp->ip_addr_str);
2428       log_error(LOG_LEVEL_CLF, "%s - - [%T] \"%s\" 400 0",
2429          csp->ip_addr_str, csp->http->cmd);
2430       return JB_ERR_PARSE;
2431    }
2432
2433    log_error(LOG_LEVEL_HEADER, "Encrypted request processed");
2434    log_applied_actions(csp->action);
2435    log_error(LOG_LEVEL_REQUEST, "https://%s%s", csp->http->hostport,
2436       csp->http->path);
2437
2438    return err;
2439
2440 }
2441
2442 /*********************************************************************
2443  *
2444  * Function    :  cgi_page_requested
2445  *
2446  * Description :  Checks if a request is for an internal CGI page.
2447  *
2448  * Parameters  :
2449  *          1  :  host = The host requested by the client.
2450  *
2451  * Returns     :  1 if a CGI page has been requested, 0 otherwise
2452  *
2453  *********************************************************************/
2454 static int cgi_page_requested(const char *host)
2455 {
2456    if ((0 == strcmpic(host, CGI_SITE_1_HOST))
2457     || (0 == strcmpic(host, CGI_SITE_1_HOST "."))
2458     || (0 == strcmpic(host, CGI_SITE_2_HOST))
2459     || (0 == strcmpic(host, CGI_SITE_2_HOST ".")))
2460    {
2461       return 1;
2462    }
2463
2464    return 0;
2465
2466 }
2467
2468
2469 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
2470 /*********************************************************************
2471  *
2472  * Function    :  continue_https_chat
2473  *
2474  * Description :  Behaves similar to chat() but only deals with
2475  *                https-inspected requests that arrive on an already
2476  *                established connection. The first request is always
2477  *                served by chat() which is a lot more complex as it
2478  *                has to deal with forwarding settings and connection
2479  *                failures etc.
2480  *
2481  *                If a connection to the server has already been
2482  *                opened it is reused unless the request is blocked
2483  *                or the forwarder changed.
2484  *
2485  *                If a connection to the server has not yet been
2486  *                opened (because the previous request was crunched),
2487  *                or the forwarder changed, the connection is dropped
2488  *                so that the client retries on a fresh one.
2489  *
2490  * Parameters  :
2491  *          1  :  csp = Current client state (buffers, headers, etc...)
2492  *
2493  * Returns     :  Nothing.
2494  *
2495  *********************************************************************/
2496 static void continue_https_chat(struct client_state *csp)
2497 {
2498    const struct forward_spec *fwd;
2499
2500    if (JB_ERR_OK != process_encrypted_request(csp))
2501    {
2502       return;
2503    }
2504
2505    csp->requests_received_total++;
2506
2507    /*
2508     * We have an encrypted request. Check if one of the crunchers wants it.
2509     */
2510    if (crunch_response_triggered(csp, crunchers_all))
2511    {
2512       /*
2513        * Yes. The client got the crunch response and we're done here.
2514        */
2515       return;
2516    }
2517    if (csp->ssl_with_server_is_opened == 0)
2518    {
2519       log_error(LOG_LEVEL_CONNECT,
2520          "Dropping the client connection on socket %d. "
2521          "The server connection has not been established yet.",
2522          csp->cfd);
2523       csp->flags &= ~CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE;
2524       return;
2525    }
2526    assert(csp->server_connection.sfd != JB_INVALID_SOCKET);
2527
2528    fwd = forward_url(csp, csp->http);
2529    if (!connection_destination_matches(&csp->server_connection, csp->http, fwd))
2530    {
2531       log_error(LOG_LEVEL_CONNECT,
2532          "Dropping the client connection on socket %d with "
2533          "server socket %d connected to %s. The forwarder has changed.",
2534          csp->cfd, csp->server_connection.sfd, csp->server_connection.host);
2535       csp->flags &= ~CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE;
2536       return;
2537    }
2538
2539    log_error(LOG_LEVEL_CONNECT,
2540       "Reusing server socket %d connected to %s. Requests already sent: %u.",
2541       csp->server_connection.sfd, csp->server_connection.host,
2542       csp->server_connection.requests_sent_total);
2543
2544    if (send_https_request(csp))
2545    {
2546       /*
2547        * Most likely the server connection timed out. We can't easily
2548        * create a new one so simply drop the client connection without a
2549        * error response to let the client retry.
2550        */
2551       log_error(LOG_LEVEL_CONNECT,
2552          "Dropping client connection on socket %d. "
2553          "Forwarding the encrypted client request failed.",
2554          csp->cfd);
2555       return;
2556    }
2557    csp->server_connection.requests_sent_total++;
2558    handle_established_connection(csp);
2559    freez(csp->receive_buffer);
2560 }
2561 #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */
2562 #endif
2563
2564
2565 /*********************************************************************
2566  *
2567  * Function    :  handle_established_connection
2568  *
2569  * Description :  Shuffle data between client and server once the
2570  *                connection has been established.
2571  *
2572  * Parameters  :
2573  *          1  :  csp = Current client state (buffers, headers, etc...)
2574  *
2575  * Returns     :  Nothing.
2576  *
2577  *********************************************************************/
2578 static void handle_established_connection(struct client_state *csp)
2579 {
2580    char *hdr;
2581    char *p;
2582    int n;
2583    struct pollfd poll_fds[2];
2584    int server_body;
2585    int ms_iis5_hack = 0;
2586    unsigned long long byte_count = 0;
2587    struct http_request *http;
2588    long len = 0; /* for buffer sizes (and negative error codes) */
2589    int buffer_and_filter_content = 0;
2590    unsigned int write_delay;
2591 #ifdef FEATURE_HTTPS_INSPECTION
2592    int ret = 0;
2593    int use_ssl_tunnel = 0;
2594    csp->dont_verify_certificate = 0;
2595
2596    if (csp->http->ssl && !(csp->action->flags & ACTION_HTTPS_INSPECTION))
2597    {
2598       /* Pass encrypted content without filtering. */
2599       use_ssl_tunnel = 1;
2600    }
2601 #endif
2602
2603    /* Skeleton for HTTP response, if we should intercept the request */
2604    struct http_response *rsp;
2605 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
2606    int watch_client_socket;
2607 #endif
2608
2609    csp->receive_buffer_size = csp->config->receive_buffer_size;
2610    csp->receive_buffer = zalloc(csp->receive_buffer_size + 1);
2611    if (csp->receive_buffer == NULL)
2612    {
2613       log_error(LOG_LEVEL_ERROR,
2614          "Out of memory. Failed to allocate the receive buffer.");
2615       rsp = cgi_error_memory();
2616       send_crunch_response(csp, rsp);
2617       return;
2618    }
2619
2620    http = csp->http;
2621
2622    /* pass data between the client and server
2623     * until one or the other shuts down the connection.
2624     */
2625
2626    server_body = 0;
2627
2628 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
2629    watch_client_socket = 0 == (csp->flags & CSP_FLAG_PIPELINED_REQUEST_WAITING);
2630 #endif
2631    write_delay = get_write_delay(csp);
2632
2633    for (;;)
2634    {
2635 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
2636       if ((csp->flags & CSP_FLAG_CHUNKED)
2637          && !(csp->flags & CSP_FLAG_CONTENT_LENGTH_SET)
2638          && ((csp->iob->eod - csp->iob->cur) >= 5)
2639          && !memcmp(csp->iob->eod-5, "0\r\n\r\n", 5))
2640       {
2641          /*
2642           * XXX: This check should be obsolete now,
2643           *      but let's wait a while to be sure.
2644           */
2645          log_error(LOG_LEVEL_CONNECT,
2646             "Looks like we got the last chunk together with "
2647             "the server headers but didn't detect it earlier. "
2648             "We better stop reading.");
2649          byte_count = (unsigned long long)(csp->iob->eod - csp->iob->cur);
2650          csp->expected_content_length = byte_count;
2651          csp->flags |= CSP_FLAG_CONTENT_LENGTH_SET;
2652       }
2653       if (server_body && server_response_is_complete(csp, byte_count))
2654       {
2655          if (csp->expected_content_length == byte_count)
2656          {
2657             log_error(LOG_LEVEL_CONNECT,
2658                "Done reading from server. Content length: %llu as expected. "
2659                "Bytes most recently read: %ld.",
2660                byte_count, len);
2661          }
2662          else
2663          {
2664             log_error(LOG_LEVEL_CONNECT,
2665                "Done reading from server. Expected content length: %llu. "
2666                "Actual content length: %llu. Bytes most recently read: %ld.",
2667                csp->expected_content_length, byte_count, len);
2668          }
2669          len = 0;
2670          /*
2671           * XXX: Should not jump around, handle_established_connection()
2672           * is complicated enough already.
2673           */
2674          goto reading_done;
2675       }
2676 #endif  /* FEATURE_CONNECTION_KEEP_ALIVE */
2677
2678       poll_fds[0].fd = csp->cfd;
2679 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
2680       if (!watch_client_socket)
2681       {
2682          /*
2683           * Ignore incoming data, but still watch out
2684           * for disconnects etc. These flags are always
2685           * implied anyway but explicitly setting them
2686           * doesn't hurt.
2687           */
2688          poll_fds[0].events = POLLERR|POLLHUP;
2689       }
2690       else
2691 #endif
2692       {
2693          poll_fds[0].events = POLLIN;
2694       }
2695       poll_fds[1].fd = csp->server_connection.sfd;
2696       poll_fds[1].events = POLLIN;
2697       n = poll(poll_fds, 2, csp->config->socket_timeout * 1000);
2698
2699       /*server or client not responding in timeout */
2700       if (n == 0)
2701       {
2702          log_error(LOG_LEVEL_CONNECT, "Socket timeout %d reached: %s",
2703             csp->config->socket_timeout, http->url);
2704          if ((byte_count == 0) && (http->ssl == 0))
2705          {
2706             send_crunch_response(csp, error_response(csp, "connection-timeout"));
2707          }
2708          mark_server_socket_tainted(csp);
2709 #ifdef FEATURE_HTTPS_INSPECTION
2710          close_client_and_server_ssl_connections(csp);
2711 #endif
2712          return;
2713       }
2714       else if (n < 0)
2715       {
2716          log_error(LOG_LEVEL_ERROR, "poll() failed!: %E");
2717          mark_server_socket_tainted(csp);
2718 #ifdef FEATURE_HTTPS_INSPECTION
2719          close_client_and_server_ssl_connections(csp);
2720 #endif
2721          return;
2722       }
2723
2724       /*
2725        * This is the body of the browser's request,
2726        * just read and write it.
2727        *
2728        * Receives data from browser and sends it to server
2729        *
2730        * XXX: Make sure the client doesn't use pipelining
2731        * behind Privoxy's back.
2732        */
2733       if ((poll_fds[0].revents & (POLLERR|POLLHUP|POLLNVAL)) != 0)
2734       {
2735          log_error(LOG_LEVEL_CONNECT,
2736             "The client socket %d has become unusable while "
2737             "the server socket %d is still open.",
2738             csp->cfd, csp->server_connection.sfd);
2739          mark_server_socket_tainted(csp);
2740          break;
2741       }
2742
2743       if (poll_fds[0].revents != 0)
2744       {
2745          int max_bytes_to_read = (int)csp->receive_buffer_size;
2746
2747 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
2748          if ((csp->flags & CSP_FLAG_CLIENT_REQUEST_COMPLETELY_READ))
2749          {
2750             if (data_is_available(csp->cfd, 0))
2751             {
2752                /*
2753                 * If the next request is already waiting, we have
2754                 * to stop poll()ing the client socket. Otherwise
2755                 * we would always return right away and get nothing
2756                 * else done.
2757                 */
2758                watch_client_socket = 0;
2759                log_error(LOG_LEVEL_CONNECT,
2760                   "Stop watching client socket %d. "
2761                   "There's already another request waiting.",
2762                   csp->cfd);
2763                continue;
2764             }
2765             /*
2766              * If the client socket is set, but there's no data
2767              * available on the socket, the client went fishing
2768              * and continuing talking to the server makes no sense.
2769              */
2770             log_error(LOG_LEVEL_CONNECT,
2771                "The client closed socket %d while "
2772                "the server socket %d is still open.",
2773                csp->cfd, csp->server_connection.sfd);
2774             mark_server_socket_tainted(csp);
2775             break;
2776          }
2777          if (csp->expected_client_content_length != 0)
2778          {
2779             if (csp->expected_client_content_length < csp->receive_buffer_size)
2780             {
2781                max_bytes_to_read = (int)csp->expected_client_content_length;
2782             }
2783             log_error(LOG_LEVEL_CONNECT,
2784                "Waiting for up to %d bytes from the client.",
2785                max_bytes_to_read);
2786          }
2787          assert(max_bytes_to_read <= csp->receive_buffer_size);
2788 #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */
2789
2790 #ifdef FEATURE_HTTPS_INSPECTION
2791          if (client_use_ssl(csp))
2792          {
2793             if (csp->http->status == 101)
2794             {
2795                len = ssl_recv_data(&(csp->ssl_client_attr),
2796                   (unsigned char *)csp->receive_buffer,
2797                   (size_t)max_bytes_to_read);
2798                if (len == -1)
2799                {
2800                   log_error(LOG_LEVEL_ERROR, "Failed to receive data "
2801                      "on client socket %d for an upgraded connection",
2802                      csp->cfd);
2803                   break;
2804                }
2805                if (len == 0)
2806                {
2807                   log_error(LOG_LEVEL_CONNECT, "Done receiving data "
2808                      "on client socket %d for an upgraded connection",
2809                      csp->cfd);
2810                   break;
2811                }
2812                byte_count += (unsigned long long)len;
2813                len = ssl_send_data(&(csp->ssl_server_attr),
2814                   (unsigned char *)csp->receive_buffer, (size_t)len);
2815                if (len == -1)
2816                {
2817                   log_error(LOG_LEVEL_ERROR, "Failed to send data "
2818                      "on server socket %d for an upgraded connection",
2819                      csp->server_connection.sfd);
2820                   break;
2821                }
2822                continue;
2823             }
2824             log_error(LOG_LEVEL_CONNECT, "Breaking with TLS/SSL.");
2825             break;
2826          }
2827          else
2828 #endif /* def FEATURE_HTTPS_INSPECTION */
2829          {
2830             len = read_socket(csp->cfd, csp->receive_buffer, max_bytes_to_read);
2831
2832             if (len <= 0)
2833             {
2834                /* XXX: not sure if this is necessary. */
2835                mark_server_socket_tainted(csp);
2836                break; /* "game over, man" */
2837             }
2838
2839 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
2840             if (csp->expected_client_content_length != 0)
2841             {
2842                assert(len <= max_bytes_to_read);
2843                csp->expected_client_content_length -= (unsigned)len;
2844                log_error(LOG_LEVEL_CONNECT,
2845                   "Expected client content length set to %llu "
2846                   "after reading %ld bytes.",
2847                   csp->expected_client_content_length, len);
2848                if (csp->expected_client_content_length == 0)
2849                {
2850                   log_error(LOG_LEVEL_CONNECT,
2851                      "Done reading from the client.");
2852                   csp->flags |= CSP_FLAG_CLIENT_REQUEST_COMPLETELY_READ;
2853                }
2854             }
2855 #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */
2856
2857             if (write_socket(csp->server_connection.sfd, csp->receive_buffer, (size_t)len))
2858             {
2859                log_error(LOG_LEVEL_ERROR, "write to: %s failed: %E", http->host);
2860                mark_server_socket_tainted(csp);
2861                return;
2862             }
2863          }
2864          continue;
2865       }
2866
2867       /*
2868        * The server wants to talk. It could be the header or the body.
2869        * If `hdr' is null, then it's the header otherwise it's the body.
2870        * FIXME: Does `hdr' really mean `host'? No.
2871        */
2872       if (poll_fds[1].revents != 0)
2873       {
2874 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
2875          /*
2876           * If we are buffering content, we don't want to eat up to
2877           * buffer-limit bytes if the client no longer cares about them.
2878           * If we aren't buffering, however, a dead client socket will be
2879           * noticed pretty much right away anyway, so we can reduce the
2880           * overhead by skipping the check.
2881           */
2882          if (buffer_and_filter_content && !socket_is_still_alive(csp->cfd))
2883          {
2884 #ifdef _WIN32
2885             log_error(LOG_LEVEL_CONNECT,
2886                "The server still wants to talk, but the client may already have hung up on us.");
2887 #else
2888             log_error(LOG_LEVEL_CONNECT,
2889                "The server still wants to talk, but the client hung up on us.");
2890             mark_server_socket_tainted(csp);
2891 #ifdef FEATURE_HTTPS_INSPECTION
2892             close_client_and_server_ssl_connections(csp);
2893 #endif
2894             return;
2895 #endif /* def _WIN32 */
2896          }
2897 #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */
2898
2899 #ifdef FEATURE_HTTPS_INSPECTION
2900          /*
2901           * Reading data from standard or secured connection (HTTP/HTTPS)
2902           */
2903          if (server_use_ssl(csp))
2904          {
2905             len = ssl_recv_data(&(csp->ssl_server_attr),
2906                (unsigned char *)csp->receive_buffer, csp->receive_buffer_size);
2907          }
2908          else
2909 #endif
2910          {
2911             len = read_socket(csp->server_connection.sfd, csp->receive_buffer,
2912                (int)csp->receive_buffer_size);
2913          }
2914
2915          if (len < 0)
2916          {
2917             log_error(LOG_LEVEL_ERROR, "read from: %s failed: %E", http->host);
2918
2919             if ((http->ssl && (csp->fwd == NULL))
2920 #ifdef FEATURE_HTTPS_INSPECTION
2921                && use_ssl_tunnel
2922 #endif
2923                 )
2924             {
2925                /*
2926                 * Just hang up. We already confirmed the client's CONNECT
2927                 * request with status code 200 and unencrypted content is
2928                 * no longer welcome.
2929                 */
2930                log_error(LOG_LEVEL_ERROR,
2931                   "CONNECT already confirmed. Unable to tell the client about the problem.");
2932                return;
2933             }
2934             else if (byte_count)
2935             {
2936                /*
2937                 * Just hang up. We already transmitted the original headers
2938                 * and parts of the original content and therefore missed the
2939                 * chance to send an error message (without risking data corruption).
2940                 *
2941                 * XXX: we could retry with a fancy range request here.
2942                 */
2943                log_error(LOG_LEVEL_ERROR, "Already forwarded the original headers. "
2944                   "Unable to tell the client about the problem.");
2945                mark_server_socket_tainted(csp);
2946 #ifdef FEATURE_HTTPS_INSPECTION
2947                close_client_and_server_ssl_connections(csp);
2948 #endif
2949                return;
2950             }
2951             /*
2952              * XXX: Consider handling the cases above the same.
2953              */
2954             mark_server_socket_tainted(csp);
2955             len = 0;
2956          }
2957
2958 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
2959          if (csp->flags & CSP_FLAG_CHUNKED)
2960          {
2961             if ((len >= 5) && !memcmp(csp->receive_buffer+len-5, "0\r\n\r\n", 5))
2962             {
2963                /* XXX: this is a temporary hack */
2964                log_error(LOG_LEVEL_CONNECT,
2965                   "Looks like we reached the end of the last chunk. "
2966                   "We better stop reading.");
2967                csp->expected_content_length = byte_count + (unsigned long long)len;
2968                csp->flags |= CSP_FLAG_CONTENT_LENGTH_SET;
2969             }
2970          }
2971          reading_done:
2972 #endif  /* FEATURE_CONNECTION_KEEP_ALIVE */
2973
2974          /*
2975           * This is guaranteed by allocating with zalloc_or_die()
2976           * and never (intentionally) writing to the last byte.
2977           *
2978           * csp->receive_buffer_size is the size of the part of the
2979           * buffer we intentionally write to, but we actually
2980           * allocated csp->receive_buffer_size+1 bytes so the assertion
2981           * stays within the allocated range.
2982           */
2983          assert(csp->receive_buffer[csp->receive_buffer_size] == '\0');
2984
2985          /*
2986           * Add a trailing zero to let be able to use string operations.
2987           * XXX: do we still need this with filter_popups gone?
2988           */
2989          assert(len <= csp->receive_buffer_size);
2990          csp->receive_buffer[len] = '\0';
2991
2992          /*
2993           * Normally, this would indicate that we've read
2994           * as much as the server has sent us and we can
2995           * close the client connection.  However, Microsoft
2996           * in its wisdom has released IIS/5 with a bug that
2997           * prevents it from sending the trailing \r\n in
2998           * a 302 redirect header (and possibly other headers).
2999           * To work around this if we've haven't parsed
3000           * a full header we'll append a trailing \r\n
3001           * and see if this now generates a valid one.
3002           *
3003           * This hack shouldn't have any impacts.  If we've
3004           * already transmitted the header or if this is a
3005           * SSL connection, then we won't bother with this
3006           * hack.  So we only work on partially received
3007           * headers.  If we append a \r\n and this still
3008           * doesn't generate a valid header, then we won't
3009           * transmit anything to the client.
3010           */
3011          if (len == 0)
3012          {
3013
3014             if (server_body || (http->ssl
3015 #ifdef FEATURE_HTTPS_INSPECTION
3016                   && use_ssl_tunnel
3017 #endif
3018                ))
3019             {
3020                /*
3021                 * If we have been buffering up the document,
3022                 * now is the time to apply content modification
3023                 * and send the result to the client.
3024                 */
3025                if (buffer_and_filter_content)
3026                {
3027                   p = execute_content_filters(csp);
3028                   /*
3029                    * If content filtering fails, use the original
3030                    * buffer and length.
3031                    * (see p != NULL ? p : csp->iob->cur below)
3032                    */
3033                   if (NULL == p)
3034                   {
3035                      csp->content_length = (size_t)(csp->iob->eod - csp->iob->cur);
3036                   }
3037 #ifdef FEATURE_COMPRESSION
3038                   else if ((csp->flags & CSP_FLAG_CLIENT_SUPPORTS_DEFLATE)
3039                      && (csp->content_length > LOWER_LENGTH_LIMIT_FOR_COMPRESSION))
3040                   {
3041                      char *compressed_content = compress_buffer(p,
3042                         (size_t *)&csp->content_length, csp->config->compression_level);
3043                      if (compressed_content != NULL)
3044                      {
3045                         freez(p);
3046                         p = compressed_content;
3047                         csp->flags |= CSP_FLAG_BUFFERED_CONTENT_DEFLATED;
3048                      }
3049                   }
3050 #endif
3051
3052                   if (JB_ERR_OK != update_server_headers(csp))
3053                   {
3054                      log_error(LOG_LEVEL_FATAL,
3055                         "Failed to update server headers. after filtering.");
3056                   }
3057
3058                   hdr = list_to_text(csp->headers);
3059                   if (hdr == NULL)
3060                   {
3061                      /* FIXME Should handle error properly */
3062                      log_error(LOG_LEVEL_FATAL, "Out of memory parsing server header");
3063                   }
3064
3065 #ifdef FEATURE_HTTPS_INSPECTION
3066                   /*
3067                    * Sending data with standard or secured connection (HTTP/HTTPS)
3068                    */
3069                   if (client_use_ssl(csp))
3070                   {
3071                      if ((ssl_send_data_delayed(&(csp->ssl_client_attr),
3072                               (const unsigned char *)hdr, strlen(hdr),
3073                               get_write_delay(csp)) < 0)
3074                         || (ssl_send_data_delayed(&(csp->ssl_client_attr),
3075                               (const unsigned char *) ((p != NULL) ? p : csp->iob->cur),
3076                               csp->content_length, get_write_delay(csp)) < 0))
3077                      {
3078                         log_error(LOG_LEVEL_ERROR, "write modified content to "
3079                            "client over TLS/SSL failed");
3080                         freez(hdr);
3081                         freez(p);
3082                         mark_server_socket_tainted(csp);
3083                         close_client_and_server_ssl_connections(csp);
3084                         return;
3085                      }
3086                   }
3087                   else
3088 #endif /* def FEATURE_HTTPS_INSPECTION */
3089                   {
3090                      if (write_socket_delayed(csp->cfd, hdr, strlen(hdr), write_delay)
3091                       || write_socket_delayed(csp->cfd, ((p != NULL) ? p : csp->iob->cur),
3092                          (size_t)csp->content_length, write_delay))
3093                      {
3094                         log_error(LOG_LEVEL_ERROR, "write modified content to client failed: %E");
3095                         freez(hdr);
3096                         freez(p);
3097                         mark_server_socket_tainted(csp);
3098                         return;
3099                      }
3100                   }
3101
3102                   freez(hdr);
3103                   freez(p);
3104                }
3105
3106                break; /* "game over, man" */
3107             }
3108
3109             /*
3110              * This is NOT the body, so
3111              * Let's pretend the server just sent us a blank line.
3112              */
3113             snprintf(csp->receive_buffer, csp->receive_buffer_size, "\r\n");
3114             len = (int)strlen(csp->receive_buffer);
3115
3116             /*
3117              * Now, let the normal header parsing algorithm below do its
3118              * job.  If it fails, we'll exit instead of continuing.
3119              */
3120
3121             ms_iis5_hack = 1;
3122          }
3123
3124          /*
3125           * If we're in the body of the server document, just write it to
3126           * the client, unless we need to buffer the body for later
3127           * content-filtering.
3128           */
3129          if (server_body || (http->ssl
3130 #ifdef FEATURE_HTTPS_INSPECTION
3131                && use_ssl_tunnel
3132 #endif
3133             ))
3134          {
3135             if (buffer_and_filter_content)
3136             {
3137                /*
3138                 * If there is no memory left for buffering the content, or the buffer limit
3139                 * has been reached, switch to non-filtering mode, i.e. make & write the
3140                 * header, flush the iob and buf, and get out of the way.
3141                 */
3142                if (add_to_iob(csp->iob, csp->config->buffer_limit, csp->receive_buffer, len))
3143                {
3144                   size_t hdrlen;
3145                   long flushed;
3146
3147                   log_error(LOG_LEVEL_INFO,
3148                      "Flushing header and buffers. Stepping back from filtering.");
3149
3150                   hdr = list_to_text(csp->headers);
3151                   if (hdr == NULL)
3152                   {
3153                      /*
3154                       * Memory is too tight to even generate the header.
3155                       * Send our static "Out-of-memory" page.
3156                       */
3157                      log_error(LOG_LEVEL_ERROR, "Out of memory while trying to flush.");
3158                      rsp = cgi_error_memory();
3159                      send_crunch_response(csp, rsp);
3160                      mark_server_socket_tainted(csp);
3161 #ifdef FEATURE_HTTPS_INSPECTION
3162                      close_client_and_server_ssl_connections(csp);
3163 #endif
3164                      return;
3165                   }
3166                   hdrlen = strlen(hdr);
3167
3168 #ifdef FEATURE_HTTPS_INSPECTION
3169                   /*
3170                    * Sending data with standard or secured connection (HTTP/HTTPS)
3171                    */
3172                   if (client_use_ssl(csp))
3173                   {
3174                      if ((ssl_send_data_delayed(&(csp->ssl_client_attr),
3175                              (const unsigned char *)hdr, hdrlen, get_write_delay(csp)) < 0)
3176                         || ((flushed = ssl_flush_socket(&(csp->ssl_client_attr),
3177                                 csp->iob)) < 0)
3178                         || (ssl_send_data_delayed(&(csp->ssl_client_attr),
3179                               (const unsigned char *)csp->receive_buffer, (size_t)len,
3180                               get_write_delay(csp)) < 0))
3181                      {
3182                         log_error(LOG_LEVEL_CONNECT,
3183                            "Flush header and buffers to client failed");
3184                         freez(hdr);
3185                         mark_server_socket_tainted(csp);
3186                         close_client_and_server_ssl_connections(csp);
3187                         return;
3188                      }
3189                   }
3190                   else
3191 #endif /* def FEATURE_HTTPS_INSPECTION */
3192                   {
3193                      if (write_socket_delayed(csp->cfd, hdr, hdrlen, write_delay)
3194                       || ((flushed = flush_iob(csp->cfd, csp->iob, write_delay)) < 0)
3195                       || write_socket_delayed(csp->cfd, csp->receive_buffer, (size_t)len,
3196                             write_delay))
3197                      {
3198                         log_error(LOG_LEVEL_CONNECT,
3199                            "Flush header and buffers to client failed: %E");
3200                         freez(hdr);
3201                         mark_server_socket_tainted(csp);
3202                         return;
3203                      }
3204                   }
3205
3206                   /*
3207                    * Reset the byte_count to the amount of bytes
3208                    * we just flushed. len will be added a few lines below,
3209                    * hdrlen doesn't matter for LOG_LEVEL_CLF.
3210                    */
3211                   byte_count = (unsigned long long)flushed;
3212                   freez(hdr);
3213                   buffer_and_filter_content = 0;
3214                   server_body = 1;
3215                }
3216             }
3217             else
3218             {
3219 #ifdef FEATURE_HTTPS_INSPECTION
3220                /*
3221                 * Sending data with standard or secured connection (HTTP/HTTPS)
3222                 */
3223                if (client_use_ssl(csp))
3224                {
3225                   ret = ssl_send_data_delayed(&(csp->ssl_client_attr),
3226                      (const unsigned char *)csp->receive_buffer, (size_t)len,
3227                      get_write_delay(csp));
3228                   if (ret < 0)
3229                   {
3230                      log_error(LOG_LEVEL_ERROR,
3231                         "Sending data to client failed");
3232                      mark_server_socket_tainted(csp);
3233                      close_client_and_server_ssl_connections(csp);
3234                      return;
3235                   }
3236                }
3237                else
3238 #endif /* def FEATURE_HTTPS_INSPECTION */
3239                {
3240                   if (write_socket_delayed(csp->cfd, csp->receive_buffer,
3241                         (size_t)len, write_delay))
3242                   {
3243                      log_error(LOG_LEVEL_ERROR, "write to client failed: %E");
3244                      mark_server_socket_tainted(csp);
3245                      return;
3246                   }
3247                }
3248             }
3249             byte_count += (unsigned long long)len;
3250             continue;
3251          }
3252          else
3253          {
3254             /*
3255              * We're still looking for the end of the server's header.
3256              * Buffer up the data we just read.  If that fails, there's
3257              * little we can do but send our static out-of-memory page.
3258              */
3259             if (add_to_iob(csp->iob, csp->config->buffer_limit, csp->receive_buffer, len))
3260             {
3261                log_error(LOG_LEVEL_ERROR, "Out of memory while looking for end of server headers.");
3262                rsp = cgi_error_memory();
3263                send_crunch_response(csp, rsp);
3264                mark_server_socket_tainted(csp);
3265 #ifdef FEATURE_HTTPS_INSPECTION
3266                close_client_and_server_ssl_connections(csp);
3267 #endif
3268                return;
3269             }
3270
3271             /* Convert iob into something sed() can digest */
3272             if (JB_ERR_PARSE == get_server_headers(csp))
3273             {
3274                if (ms_iis5_hack)
3275                {
3276                   /*
3277                    * Well, we tried our MS IIS/5 hack and it didn't work.
3278                    * The header is incomplete and there isn't anything
3279                    * we can do about it.
3280                    */
3281                   log_error(LOG_LEVEL_ERROR, "Invalid server headers. "
3282                      "Applying the MS IIS5 hack didn't help.");
3283                   log_error(LOG_LEVEL_CLF,
3284                      "%s - - [%T] \"%s\" 502 0", csp->ip_addr_str, http->cmd);
3285 #ifdef FEATURE_HTTPS_INSPECTION
3286                   /*
3287                    * Sending data with standard or secured connection (HTTP/HTTPS)
3288                    */
3289                   if (client_use_ssl(csp))
3290                   {
3291                      ssl_send_data_delayed(&(csp->ssl_client_attr),
3292                         (const unsigned char *)INVALID_SERVER_HEADERS_RESPONSE,
3293                         strlen(INVALID_SERVER_HEADERS_RESPONSE), get_write_delay(csp));
3294                   }
3295                   else
3296 #endif /* def FEATURE_HTTPS_INSPECTION */
3297                   {
3298                      write_socket_delayed(csp->cfd,
3299                         INVALID_SERVER_HEADERS_RESPONSE,
3300                         strlen(INVALID_SERVER_HEADERS_RESPONSE), write_delay);
3301                   }
3302                   mark_server_socket_tainted(csp);
3303 #ifdef FEATURE_HTTPS_INSPECTION
3304                   close_client_and_server_ssl_connections(csp);
3305 #endif
3306                   return;
3307                }
3308                else
3309                {
3310                   /*
3311                    * Since we have to wait for more from the server before
3312                    * we can parse the headers we just continue here.
3313                    */
3314                   log_error(LOG_LEVEL_CONNECT,
3315                      "Continuing buffering server headers from socket %d. "
3316                      "Bytes most recently read: %ld.", csp->cfd, len);
3317                   continue;
3318                }
3319             }
3320             else
3321             {
3322                /*
3323                 * Account for the content bytes we
3324                 * might have gotten with the headers.
3325                 */
3326                assert(csp->iob->eod >= csp->iob->cur);
3327                byte_count = (unsigned long long)(csp->iob->eod - csp->iob->cur);
3328             }
3329
3330             /* Did we actually get anything? */
3331             if (NULL == csp->headers->first)
3332             {
3333                if ((csp->flags & CSP_FLAG_REUSED_CLIENT_CONNECTION))
3334                {
3335                   log_error(LOG_LEVEL_ERROR,
3336                      "No server or forwarder response received on socket %d. "
3337                      "Closing client socket %d without sending data.",
3338                      csp->server_connection.sfd, csp->cfd);
3339                   log_error(LOG_LEVEL_CLF,
3340                      "%s - - [%T] \"%s\" 502 0", csp->ip_addr_str, http->cmd);
3341                }
3342                else
3343                {
3344                   log_error(LOG_LEVEL_ERROR,
3345                      "No server or forwarder response received on socket %d.",
3346                      csp->server_connection.sfd);
3347                   send_crunch_response(csp, error_response(csp, "no-server-data"));
3348                }
3349                free_http_request(http);
3350                mark_server_socket_tainted(csp);
3351 #ifdef FEATURE_HTTPS_INSPECTION
3352                close_client_and_server_ssl_connections(csp);
3353 #endif
3354                return;
3355             }
3356
3357             if (!csp->headers->first->str)
3358             {
3359                log_error(LOG_LEVEL_ERROR, "header search: csp->headers->first->str == NULL, assert will be called");
3360             }
3361             assert(csp->headers->first->str);
3362
3363             if (strncmpic(csp->headers->first->str, "HTTP", 4) &&
3364                 strncmpic(csp->headers->first->str, "ICY", 3))
3365             {
3366                /*
3367                 * It doesn't look like a HTTP (or Shoutcast) response:
3368                 * tell the client and log the problem.
3369                 */
3370                if (strlen(csp->headers->first->str) > 30)
3371                {
3372                   csp->headers->first->str[30] = '\0';
3373                }
3374                log_error(LOG_LEVEL_ERROR,
3375                   "Invalid server or forwarder response. Starts with: %s",
3376                   csp->headers->first->str);
3377                log_error(LOG_LEVEL_CLF,
3378                   "%s - - [%T] \"%s\" 502 0", csp->ip_addr_str, http->cmd);
3379 #ifdef FEATURE_HTTPS_INSPECTION
3380                /*
3381                 * Sending data with standard or secured connection (HTTP/HTTPS)
3382                 */
3383                if (client_use_ssl(csp))
3384                {
3385                   ssl_send_data_delayed(&(csp->ssl_client_attr),
3386                      (const unsigned char *)INVALID_SERVER_HEADERS_RESPONSE,
3387                      strlen(INVALID_SERVER_HEADERS_RESPONSE),
3388                      get_write_delay(csp));
3389                }
3390                else
3391 #endif /* def FEATURE_HTTPS_INSPECTION */
3392                {
3393                   write_socket_delayed(csp->cfd, INVALID_SERVER_HEADERS_RESPONSE,
3394                      strlen(INVALID_SERVER_HEADERS_RESPONSE), write_delay);
3395                }
3396                free_http_request(http);
3397                mark_server_socket_tainted(csp);
3398 #ifdef FEATURE_HTTPS_INSPECTION
3399                close_client_and_server_ssl_connections(csp);
3400 #endif
3401                return;
3402             }
3403
3404             /*
3405              * We have now received the entire server header,
3406              * filter it and send the result to the client
3407              */
3408             if (JB_ERR_OK != sed(csp, FILTER_SERVER_HEADERS))
3409             {
3410                log_error(LOG_LEVEL_CLF,
3411                   "%s - - [%T] \"%s\" 502 0", csp->ip_addr_str, http->cmd);
3412 #ifdef FEATURE_HTTPS_INSPECTION
3413                /*
3414                 * Sending data with standard or secured connection (HTTP/HTTPS)
3415                 */
3416                if (client_use_ssl(csp))
3417                {
3418                   ssl_send_data_delayed(&(csp->ssl_client_attr),
3419                      (const unsigned char *)INVALID_SERVER_HEADERS_RESPONSE,
3420                      strlen(INVALID_SERVER_HEADERS_RESPONSE),
3421                      get_write_delay(csp));
3422                }
3423                else
3424 #endif
3425                {
3426                   write_socket_delayed(csp->cfd, INVALID_SERVER_HEADERS_RESPONSE,
3427                      strlen(INVALID_SERVER_HEADERS_RESPONSE), write_delay);
3428                }
3429                free_http_request(http);
3430                mark_server_socket_tainted(csp);
3431 #ifdef FEATURE_HTTPS_INSPECTION
3432                close_client_and_server_ssl_connections(csp);
3433 #endif
3434                return;
3435             }
3436             hdr = list_to_text(csp->headers);
3437             if (hdr == NULL)
3438             {
3439                /* FIXME Should handle error properly */
3440                log_error(LOG_LEVEL_FATAL, "Out of memory parsing server header");
3441             }
3442
3443             if ((csp->flags & CSP_FLAG_CHUNKED)
3444                && !(csp->flags & CSP_FLAG_CONTENT_LENGTH_SET)
3445                && ((csp->iob->eod - csp->iob->cur) >= 5)
3446                && !memcmp(csp->iob->eod-5, "0\r\n\r\n", 5))
3447             {
3448                log_error(LOG_LEVEL_CONNECT,
3449                   "Looks like we got the last chunk together with "
3450                   "the server headers. We better stop reading.");
3451                byte_count = (unsigned long long)(csp->iob->eod - csp->iob->cur);
3452                csp->expected_content_length = byte_count;
3453                csp->flags |= CSP_FLAG_CONTENT_LENGTH_SET;
3454             }
3455
3456             csp->server_connection.response_received = time(NULL);
3457
3458             if (crunch_response_triggered(csp, crunchers_light))
3459             {
3460                /*
3461                 * One of the tags created by a server-header
3462                 * tagger triggered a crunch. We already
3463                 * delivered the crunch response to the client
3464                 * and are done here after cleaning up.
3465                 */
3466                freez(hdr);
3467                mark_server_socket_tainted(csp);
3468 #ifdef FEATURE_HTTPS_INSPECTION
3469                close_client_and_server_ssl_connections(csp);
3470 #endif
3471                return;
3472             }
3473
3474             /* Buffer and pcrs filter this if appropriate. */
3475             buffer_and_filter_content = content_requires_filtering(csp);
3476
3477             if (!buffer_and_filter_content)
3478             {
3479                /*
3480                 * Write the server's (modified) header to
3481                 * the client (along with anything else that
3482                 * may be in the buffer). Use standard or secured
3483                 * connection.
3484                 */
3485 #ifdef FEATURE_HTTPS_INSPECTION
3486                if (client_use_ssl(csp))
3487                {
3488                   if ((ssl_send_data_delayed(&(csp->ssl_client_attr),
3489                           (const unsigned char *)hdr, strlen(hdr),
3490                           get_write_delay(csp)) < 0)
3491                      || (len = ssl_flush_socket(&(csp->ssl_client_attr),
3492                             csp->iob) < 0))
3493                   {
3494                      log_error(LOG_LEVEL_CONNECT, "Write header to client failed");
3495
3496                      /*
3497                       * The write failed, so don't bother mentioning it
3498                       * to the client... it probably can't hear us anyway.
3499                       */
3500                      freez(hdr);
3501                      mark_server_socket_tainted(csp);
3502 #ifdef FEATURE_HTTPS_INSPECTION
3503                      close_client_and_server_ssl_connections(csp);
3504 #endif
3505                      return;
3506                   }
3507                }
3508                else
3509 #endif /* def FEATURE_HTTPS_INSPECTION */
3510                {
3511                   if (write_socket_delayed(csp->cfd, hdr, strlen(hdr), write_delay)
3512                      || ((len = flush_iob(csp->cfd, csp->iob, write_delay)) < 0))
3513                   {
3514                      log_error(LOG_LEVEL_ERROR,
3515                         "write header to client failed");
3516                      /*
3517                       * The write failed, so don't bother mentioning it
3518                       * to the client... it probably can't hear us anyway.
3519                       */
3520                      freez(hdr);
3521                      mark_server_socket_tainted(csp);
3522                      return;
3523                   }
3524                }
3525                                 }
3526
3527             /* we're finished with the server's header */
3528
3529             freez(hdr);
3530             server_body = 1;
3531
3532             /*
3533              * If this was a MS IIS/5 hack then it means the server
3534              * has already closed the connection. Nothing more to read.
3535              * Time to bail.
3536              */
3537             if (ms_iis5_hack)
3538             {
3539                log_error(LOG_LEVEL_ERROR,
3540                   "Closed server connection detected. "
3541                   "Applying the MS IIS5 hack didn't help.");
3542                log_error(LOG_LEVEL_CLF,
3543                   "%s - - [%T] \"%s\" 502 0", csp->ip_addr_str, http->cmd);
3544 #ifdef FEATURE_HTTPS_INSPECTION
3545                /*
3546                 * Sending data with standard or secured connection (HTTP/HTTPS)
3547                 */
3548                if (client_use_ssl(csp))
3549                {
3550                   ssl_send_data_delayed(&(csp->ssl_client_attr),
3551                      (const unsigned char *)INVALID_SERVER_HEADERS_RESPONSE,
3552                      strlen(INVALID_SERVER_HEADERS_RESPONSE),
3553                      get_write_delay(csp));
3554                }
3555                else
3556 #endif /* def FEATURE_HTTPS_INSPECTION */
3557                {
3558                   write_socket_delayed(csp->cfd, INVALID_SERVER_HEADERS_RESPONSE,
3559                      strlen(INVALID_SERVER_HEADERS_RESPONSE), write_delay);
3560                }
3561                mark_server_socket_tainted(csp);
3562 #ifdef FEATURE_HTTPS_INSPECTION
3563                close_client_and_server_ssl_connections(csp);
3564 #endif
3565                return;
3566             }
3567          }
3568          continue;
3569       }
3570       mark_server_socket_tainted(csp);
3571 #ifdef FEATURE_HTTPS_INSPECTION
3572       close_client_and_server_ssl_connections(csp);
3573 #endif
3574       return; /* huh? we should never get here */
3575    }
3576
3577    if (csp->content_length == 0)
3578    {
3579       /*
3580        * If Privoxy didn't recalculate the Content-Length,
3581        * byte_count is still correct.
3582        */
3583       csp->content_length = byte_count;
3584    }
3585
3586 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
3587    if ((csp->flags & CSP_FLAG_CONTENT_LENGTH_SET)
3588       && (csp->expected_content_length != byte_count))
3589    {
3590       log_error(LOG_LEVEL_CONNECT,
3591          "Received %llu bytes while expecting %llu.",
3592          byte_count, csp->expected_content_length);
3593       mark_server_socket_tainted(csp);
3594    }
3595 #endif
3596
3597 #ifdef FEATURE_HTTPS_INSPECTION
3598    if (client_use_ssl(csp))
3599    {
3600       log_error(LOG_LEVEL_CLF, "%s - - [%T] \"%s https://%s%s %s\" 200 %llu",
3601          csp->ip_addr_str, http->gpc, http->hostport, http->path,
3602          http->version, csp->content_length);
3603    }
3604    else
3605 #endif
3606    {
3607       log_error(LOG_LEVEL_CLF, "%s - - [%T] \"%s\" 200 %llu",
3608          csp->ip_addr_str, http->ocmd, csp->content_length);
3609    }
3610    csp->server_connection.timestamp = time(NULL);
3611 }
3612
3613
3614 /*********************************************************************
3615  *
3616  * Function    :  chat
3617  *
3618  * Description :  Once a connection from the client has been accepted,
3619  *                this function is called (via serve()) to handle the
3620  *                main business of the communication.  This function
3621  *                returns after dealing with a single request. It can
3622  *                be called multiple times with the same client socket
3623  *                if the client is keeping the connection alive.
3624  *
3625  *                The decision whether or not a client connection will
3626  *                be kept alive is up to the caller which also must
3627  *                close the client socket when done.
3628  *
3629  *                FIXME: chat is nearly thousand lines long.
3630  *                Ridiculous.
3631  *
3632  * Parameters  :
3633  *          1  :  csp = Current client state (buffers, headers, etc...)
3634  *
3635  * Returns     :  Nothing.
3636  *
3637  *********************************************************************/
3638 static void chat(struct client_state *csp)
3639 {
3640    const struct forward_spec *fwd;
3641    struct http_request *http;
3642    /* Skeleton for HTTP response, if we should intercept the request */
3643    struct http_response *rsp;
3644 #ifdef FEATURE_HTTPS_INSPECTION
3645    int use_ssl_tunnel = 0;
3646 #endif
3647
3648    http = csp->http;
3649
3650    if (receive_client_request(csp) != JB_ERR_OK)
3651    {
3652       return;
3653    }
3654    if (parse_client_request(csp) != JB_ERR_OK)
3655    {
3656       return;
3657    }
3658
3659    /* decide how to route the HTTP request */
3660    fwd = forward_url(csp, http);
3661
3662 #ifdef FEATURE_HTTPS_INSPECTION
3663    /*
3664     * Setting flags to use old solution with SSL tunnel and to disable
3665     * certificate verification.
3666     */
3667    if (csp->http->ssl && !(csp->action->flags & ACTION_HTTPS_INSPECTION)
3668       && !cgi_page_requested(csp->http->host))
3669    {
3670       use_ssl_tunnel = 1;
3671    }
3672
3673    if (http->ssl && (csp->action->flags & ACTION_IGNORE_CERTIFICATE_ERRORS))
3674    {
3675       csp->dont_verify_certificate = 1;
3676    }
3677 #endif
3678
3679    /*
3680     * build the http request to send to the server
3681     * we have to do one of the following:
3682     *
3683     * create =    use the original HTTP request to create a new
3684     *             HTTP request that has either the path component
3685     *             without the http://domainspec (w/path) or the
3686     *             full orininal URL (w/url)
3687     *             Note that the path and/or the HTTP version may
3688     *             have been altered by now.
3689     *
3690     * SSL proxy = Open a socket to the host:port of the server
3691     *             and create TLS/SSL connection with server and
3692     *             with client. Then behave like mediator between
3693     *             client and server over TLS/SSL.
3694     *
3695     * SSL proxy = Pass the request unchanged if forwarding a CONNECT
3696     *    with     request to a parent proxy. Note that we'll be sending
3697     * forwarding  the CFAIL message ourselves if connecting to the parent
3698     *             fails, but we won't send a CSUCCEED message if it works,
3699     *             since that would result in a double message (ours and the
3700     *             parent's). After sending the request to the parent, we
3701     *             must parse answer and send it to client. If connection
3702     *             with server is established, we do TLS/SSL proxy. Otherwise
3703     *             we send parent response to client and close connections.
3704     *
3705     * here's the matrix:
3706     *                        SSL
3707     *                    0        1
3708     *                +--------+--------+
3709     *                |        |        |
3710     *             0  | create |   SSL  |
3711     *                | w/path |  proxy |
3712     *  Forwarding    +--------+--------+
3713     *                |        |   SSL  |
3714     *             1  | create |  proxy |
3715     *                | w/url  |+forward|
3716     *                +--------+--------+
3717     *
3718     */
3719
3720 #ifdef FEATURE_HTTPS_INSPECTION
3721    /*
3722     * Presetting SSL client and server flags
3723     */
3724    if (http->ssl && !use_ssl_tunnel)
3725    {
3726       http->client_ssl = 1;
3727       http->server_ssl = 1;
3728    }
3729    else
3730    {
3731       http->client_ssl = 0;
3732       http->server_ssl = 0;
3733    }
3734 #endif
3735
3736 #ifdef FEATURE_HTTPS_INSPECTION
3737    /*
3738     * Log the request unless we're https inspecting
3739     * in which case we don't have the path yet and
3740     * will log the request later.
3741     */
3742    if (!client_use_ssl(csp))
3743 #endif
3744    {
3745       log_error(LOG_LEVEL_REQUEST, "%s%s", http->hostport, http->path);
3746    }
3747
3748    if (http->ssl && connect_port_is_forbidden(csp))
3749    {
3750       const char *acceptable_connect_ports =
3751          csp->action->string[ACTION_STRING_LIMIT_CONNECT];
3752       assert(NULL != acceptable_connect_ports);
3753       log_error(LOG_LEVEL_INFO, "Request from %s marked for blocking. "
3754          "limit-connect{%s} doesn't allow CONNECT requests to %s",
3755          csp->ip_addr_str, acceptable_connect_ports, csp->http->hostport);
3756       csp->action->flags |= ACTION_BLOCK;
3757       http->ssl = 0;
3758 #ifdef FEATURE_HTTPS_INSPECTION
3759       http->client_ssl = 0;
3760       http->server_ssl = 0;
3761 #endif
3762    }
3763
3764
3765    freez(csp->headers->first->str);
3766    build_request_line(csp, fwd, &csp->headers->first->str);
3767
3768    /*
3769     * We have a request. Check if one of the crunchers wants it
3770     * unless the client wants to use TLS/SSL in which case we
3771     * haven't setup the TLS context yet and will send the crunch
3772     * response later.
3773     */
3774    if (
3775 #ifdef FEATURE_HTTPS_INSPECTION
3776        !client_use_ssl(csp) &&
3777 #endif
3778        crunch_response_triggered(csp, crunchers_all))
3779    {
3780       /*
3781        * Yes. The client got the crunch response and we're done here.
3782        */
3783       return;
3784    }
3785
3786    log_applied_actions(csp->action);
3787    if (fwd->forward_host)
3788    {
3789       log_error(LOG_LEVEL_CONNECT, "via [%s]:%d to: %s",
3790          fwd->forward_host, fwd->forward_port, http->hostport);
3791    }
3792    else
3793    {
3794       log_error(LOG_LEVEL_CONNECT, "to %s", http->hostport);
3795    }
3796
3797    /* here we connect to the server, gateway, or the forwarder */
3798
3799 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
3800    if ((csp->server_connection.sfd != JB_INVALID_SOCKET)
3801       && socket_is_still_alive(csp->server_connection.sfd)
3802       && connection_destination_matches(&csp->server_connection, http, fwd))
3803    {
3804       log_error(LOG_LEVEL_CONNECT,
3805          "Reusing server socket %d connected to %s. Total requests: %u.",
3806          csp->server_connection.sfd, csp->server_connection.host,
3807          csp->server_connection.requests_sent_total);
3808    }
3809    else
3810    {
3811       if (csp->server_connection.sfd != JB_INVALID_SOCKET)
3812       {
3813 #ifdef FEATURE_CONNECTION_SHARING
3814          if (csp->config->feature_flags & RUNTIME_FEATURE_CONNECTION_SHARING
3815 #ifdef FEATURE_HTTPS_INSPECTION
3816             && !server_use_ssl(csp)
3817 #endif
3818             )
3819          {
3820             remember_connection(&csp->server_connection);
3821          }
3822          else
3823 #endif /* def FEATURE_CONNECTION_SHARING */
3824          {
3825             log_error(LOG_LEVEL_CONNECT,
3826                "Closing server socket %d connected to %s. Total requests: %u.",
3827                csp->server_connection.sfd, csp->server_connection.host,
3828                csp->server_connection.requests_sent_total);
3829             close_socket(csp->server_connection.sfd);
3830          }
3831          mark_connection_closed(&csp->server_connection);
3832       }
3833 #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */
3834 #ifdef FEATURE_HTTPS_INSPECTION
3835       if (http->ssl && !use_ssl_tunnel)
3836       {
3837          int ret;
3838          /*
3839           * Creating a SSL proxy.
3840           *
3841           * By sending the CSUCCEED message we're lying to the client as
3842           * the connection hasn't actually been established yet. We don't
3843           * establish the connection until we have seen and parsed the
3844           * encrypted client headers.
3845           */
3846          if (write_socket_delayed(csp->cfd, CSUCCEED,
3847                strlen(CSUCCEED), get_write_delay(csp)) != 0)
3848          {
3849             log_error(LOG_LEVEL_ERROR, "Sending SUCCEED to client failed");
3850             return;
3851          }
3852
3853          ret = create_client_ssl_connection(csp);
3854          if (ret != 0)
3855          {
3856             log_error(LOG_LEVEL_ERROR,
3857                "Failed to open a secure connection with the client");
3858             return;
3859          }
3860          if (JB_ERR_OK != process_encrypted_request(csp))
3861          {
3862             close_client_ssl_connection(csp);
3863             return;
3864          }
3865          /*
3866           * We have an encrypted request. Check if one of the crunchers now
3867           * wants it (for example because the previously invisible path was
3868           * required to match).
3869           */
3870          if (crunch_response_triggered(csp, crunchers_all))
3871          {
3872             /*
3873              * Yes. The client got the crunch response and we're done here.
3874              */
3875             return;
3876          }
3877       }
3878 #endif
3879       /*
3880        * Connecting to destination server
3881        */
3882       csp->server_connection.sfd = forwarded_connect(fwd, http, csp);
3883
3884       if (csp->server_connection.sfd == JB_INVALID_SOCKET)
3885       {
3886          if (fwd->type != SOCKS_NONE)
3887          {
3888             /* Socks error. */
3889             rsp = error_response(csp, "forwarding-failed");
3890          }
3891          else if (errno == EINVAL)
3892          {
3893             rsp = error_response(csp, "no-such-domain");
3894          }
3895          else
3896          {
3897             rsp = error_response(csp, "connect-failed");
3898          }
3899
3900          /* Write the answer to the client */
3901          if (rsp != NULL)
3902          {
3903             send_crunch_response(csp, rsp);
3904          }
3905
3906          /*
3907           * Temporary workaround to prevent already-read client
3908           * bodies from being parsed as new requests. For now we
3909           * err on the safe side and throw all the following
3910           * requests under the bus, even if no client body has been
3911           * buffered. A compliant client will repeat the dropped
3912           * requests on an untainted connection.
3913           *
3914           * The proper fix is to discard the no longer needed
3915           * client body in the buffer (if there is one) and to
3916           * continue parsing the bytes that follow.
3917           */
3918 #ifdef FEATURE_HTTPS_INSPECTION
3919          close_client_ssl_connection(csp);
3920 #endif
3921          drain_and_close_socket(csp->cfd);
3922          csp->cfd = JB_INVALID_SOCKET;
3923
3924          return;
3925       }
3926
3927 #ifdef FEATURE_HTTPS_INSPECTION
3928       /*
3929        * Creating TLS/SSL connections with destination server or parent
3930        * proxy. If forwarding is enabled, we must send client request to
3931        * parent proxy and receive, parse and resend parent proxy answer.
3932        */
3933       if (http->ssl && !use_ssl_tunnel)
3934       {
3935          if (fwd->forward_host != NULL)
3936          {
3937             char server_response[BUFFER_SIZE];
3938             int ret = 0;
3939             int len = 0;
3940             char *hdr = list_to_text(csp->headers);
3941             memset(server_response, 0, sizeof(server_response));
3942
3943             if (hdr == NULL)
3944             {
3945                log_error(LOG_LEVEL_FATAL,
3946                   "Out of memory parsing client header");
3947             }
3948             list_remove_all(csp->headers);
3949
3950             /*
3951              * Sending client's CONNECT request to the parent proxy
3952              */
3953             ret = write_socket(csp->server_connection.sfd, hdr, strlen(hdr));
3954
3955             freez(hdr);
3956
3957             if (ret != 0)
3958             {
3959                log_error(LOG_LEVEL_CONNECT,
3960                   "Sending request headers to: %s failed", http->hostport);
3961                mark_server_socket_tainted(csp);
3962                close_client_ssl_connection(csp);
3963                return;
3964             }
3965
3966             /* Waiting for parent proxy server response */
3967             len = read_socket(csp->server_connection.sfd, server_response,
3968                sizeof(server_response)-1);
3969
3970             if (len <= 0)
3971             {
3972                log_error(LOG_LEVEL_ERROR, "No response from parent proxy "
3973                   "server on socket %d.", csp->server_connection.sfd);
3974
3975                rsp = error_response(csp, "no-server-data");
3976                if (rsp)
3977                {
3978                   send_crunch_response(csp, rsp);
3979                }
3980                mark_server_socket_tainted(csp);
3981                close_client_ssl_connection(csp);
3982                return;
3983             }
3984
3985             /*
3986              * Test if the connection to the destination server was
3987              * established successfully by the parent proxy.
3988              */
3989             if (!tunnel_established_successfully(server_response, (unsigned int)len))
3990             {
3991                log_error(LOG_LEVEL_ERROR,
3992                   "The forwarder %s failed to establish a connection with %s",
3993                   fwd->forward_host, http->host);
3994                rsp = error_response(csp, "connect-failed");
3995                if (rsp)
3996                {
3997                   send_crunch_response(csp, rsp);
3998                }
3999                mark_server_socket_tainted(csp);
4000                close_client_ssl_connection(csp);
4001                return;
4002             }
4003          } /* -END- if (fwd->forward_host != NULL) */
4004
4005          /*
4006           * We can now create the TLS/SSL connection with the destination server.
4007           */
4008          int ret = create_server_ssl_connection(csp);
4009          if (ret != 0)
4010          {
4011             if (csp->server_cert_verification_result != SSL_CERT_VALID &&
4012                 csp->server_cert_verification_result != SSL_CERT_NOT_VERIFIED)
4013             {
4014                /*
4015                 * If the server certificate is invalid, we must inform
4016                 * the client and then close connection to the client.
4017                 */
4018                ssl_send_certificate_error(csp);
4019                close_client_and_server_ssl_connections(csp);
4020                return;
4021             }
4022             if (csp->server_cert_verification_result == SSL_CERT_NOT_VERIFIED
4023              || csp->server_cert_verification_result == SSL_CERT_VALID)
4024             {
4025                /*
4026                 * The TLS/SSL connection wasn't created but an invalid
4027                 * certificate wasn't detected. Report it as connection
4028                 * failure.
4029                 */
4030                rsp = error_response(csp, "connect-failed");
4031                if (rsp)
4032                {
4033                   send_crunch_response(csp, rsp);
4034                }
4035                close_client_and_server_ssl_connections(csp);
4036                return;
4037             }
4038          }
4039       }/* -END- if (http->ssl) */
4040 #endif /* def FEATURE_HTTPS_INSPECTION */
4041
4042 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
4043       save_connection_destination(csp->server_connection.sfd,
4044          http, fwd, &csp->server_connection);
4045       csp->server_connection.keep_alive_timeout =
4046          (unsigned)csp->config->keep_alive_timeout;
4047    }
4048 #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */
4049
4050    csp->server_connection.requests_sent_total++;
4051
4052    if ((fwd->type == SOCKS_5T) && (NULL == csp->headers->first))
4053    {
4054       /* Client headers have been sent optimistically */
4055       assert(csp->headers->last == NULL);
4056    }
4057    else if (http->ssl == 0 || (fwd->forward_host
4058 #ifdef FEATURE_HTTPS_INSPECTION
4059          && use_ssl_tunnel
4060 #endif
4061            ))
4062    {
4063       if (send_http_request(csp))
4064       {
4065          rsp = error_response(csp, "connect-failed");
4066          if (rsp)
4067          {
4068             send_crunch_response(csp, rsp);
4069          }
4070          return;
4071       }
4072    }
4073    else
4074    {
4075       /*
4076        * Using old solution with SSL tunnel or new solution with SSL proxy
4077        */
4078       list_remove_all(csp->headers);
4079 #ifdef FEATURE_HTTPS_INSPECTION
4080       if (use_ssl_tunnel)
4081 #endif
4082       {
4083          /*
4084          * We're running an SSL tunnel and we're not forwarding,
4085          * so just ditch the client headers, send the "connect succeeded"
4086          * message to the client, flush the rest, and get out of the way.
4087          */
4088          if (write_socket_delayed(csp->cfd, CSUCCEED,
4089                strlen(CSUCCEED), get_write_delay(csp)))
4090          {
4091             return;
4092          }
4093       }
4094 #ifdef FEATURE_HTTPS_INSPECTION
4095       else
4096       {
4097          /*
4098           * If server certificate has been verified and is invalid,
4099           * we must inform the client and then close the connection
4100           * with client and server.
4101           */
4102          if (csp->server_cert_verification_result != SSL_CERT_VALID &&
4103              csp->server_cert_verification_result != SSL_CERT_NOT_VERIFIED)
4104          {
4105             ssl_send_certificate_error(csp);
4106             close_client_and_server_ssl_connections(csp);
4107             return;
4108          }
4109          if (send_https_request(csp))
4110          {
4111             rsp = error_response(csp, "connect-failed");
4112             if (rsp)
4113             {
4114                send_crunch_response(csp, rsp);
4115             }
4116             close_client_and_server_ssl_connections(csp);
4117             return;
4118          }
4119       }
4120 #endif /* def FEATURE_HTTPS_INSPECTION */
4121       clear_iob(csp->client_iob);
4122    }/* -END- else ... if (http->ssl == 1) */
4123
4124    log_error(LOG_LEVEL_CONNECT, "to %s successful", http->hostport);
4125
4126    /* XXX: should the time start earlier for optimistically sent data? */
4127    csp->server_connection.request_sent = time(NULL);
4128
4129    handle_established_connection(csp);
4130    freez(csp->receive_buffer);
4131 }
4132
4133
4134 #ifdef FUZZ
4135 /*********************************************************************
4136  *
4137  * Function    :  fuzz_server_response
4138  *
4139  * Description :  Treat the input as a whole server response.
4140  *
4141  * Parameters  :
4142  *          1  :  csp = Current client state (buffers, headers, etc...)
4143  *          2  :  fuzz_input_file = File to read the input from.
4144  *
4145  * Returns     :  0
4146  *
4147  *********************************************************************/
4148 extern int fuzz_server_response(struct client_state *csp, char *fuzz_input_file)
4149 {
4150    static struct forward_spec fwd; /* Zero'd due to being static */
4151    csp->cfd = 0;
4152
4153    if (strcmp(fuzz_input_file, "-") == 0)
4154    {
4155       /* XXX: Doesn't work yet. */
4156       csp->server_connection.sfd = 0;
4157    }
4158    else
4159    {
4160       csp->server_connection.sfd = open(fuzz_input_file, O_RDONLY);
4161       if (csp->server_connection.sfd == -1)
4162       {
4163          log_error(LOG_LEVEL_FATAL, "Failed to open %s: %E",
4164             fuzz_input_file);
4165       }
4166    }
4167    csp->fwd = &fwd;
4168    csp->content_type |= CT_GIF;
4169    csp->action->flags |= ACTION_DEANIMATE;
4170    csp->action->string[ACTION_STRING_DEANIMATE] = "last";
4171
4172    csp->http->path = strdup_or_die("/");
4173    csp->http->host = strdup_or_die("fuzz.example.org");
4174    csp->http->hostport = strdup_or_die("fuzz.example.org:80");
4175    /* Prevent client socket monitoring */
4176    csp->flags |= CSP_FLAG_PIPELINED_REQUEST_WAITING;
4177    csp->flags |= CSP_FLAG_CHUNKED;
4178
4179    csp->config->feature_flags |= RUNTIME_FEATURE_CONNECTION_KEEP_ALIVE;
4180    csp->flags |= CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE;
4181
4182    csp->content_type |= CT_DECLARED|CT_GIF;
4183
4184    csp->config->socket_timeout = 0;
4185
4186    cgi_init_error_messages();
4187
4188    handle_established_connection(csp);
4189    freez(csp->receive_buffer);
4190
4191    return 0;
4192 }
4193 #endif
4194
4195
4196 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
4197 /*********************************************************************
4198  *
4199  * Function    :  prepare_csp_for_next_request
4200  *
4201  * Description :  Put the csp in a mostly vergin state.
4202  *
4203  * Parameters  :
4204  *          1  :  csp = Current client state (buffers, headers, etc...)
4205  *
4206  * Returns     :  N/A
4207  *
4208  *********************************************************************/
4209 static void prepare_csp_for_next_request(struct client_state *csp)
4210 {
4211    csp->content_type = 0;
4212    csp->content_length = 0;
4213    csp->expected_content_length = 0;
4214    csp->expected_client_content_length = 0;
4215    list_remove_all(csp->headers);
4216    clear_iob(csp->iob);
4217    freez(csp->error_message);
4218    free_http_request(csp->http);
4219    destroy_list(csp->headers);
4220 #ifdef FEATURE_HTTPS_INSPECTION
4221    destroy_list(csp->https_headers);
4222 #endif
4223    destroy_list(csp->tags);
4224 #ifdef FEATURE_CLIENT_TAGS
4225    destroy_list(csp->client_tags);
4226    freez(csp->client_address);
4227 #endif
4228    free_current_action(csp->action);
4229    if (NULL != csp->fwd)
4230    {
4231       unload_forward_spec(csp->fwd);
4232       csp->fwd = NULL;
4233    }
4234    /* XXX: Store per-connection flags someplace else. */
4235    csp->flags = (CSP_FLAG_ACTIVE | CSP_FLAG_REUSED_CLIENT_CONNECTION);
4236 #ifdef FEATURE_TOGGLE
4237    if (global_toggle_state)
4238 #endif /* def FEATURE_TOGGLE */
4239    {
4240       csp->flags |= CSP_FLAG_TOGGLED_ON;
4241    }
4242
4243    if (csp->client_iob->eod > csp->client_iob->cur)
4244    {
4245       long bytes_to_shift = csp->client_iob->cur - csp->client_iob->buf;
4246       size_t data_length  = (size_t)(csp->client_iob->eod - csp->client_iob->cur);
4247
4248       assert(bytes_to_shift > 0);
4249       assert(data_length > 0);
4250
4251       log_error(LOG_LEVEL_CONNECT, "Shifting %lu pipelined bytes by %ld bytes",
4252          data_length, bytes_to_shift);
4253       memmove(csp->client_iob->buf, csp->client_iob->cur, data_length);
4254       csp->client_iob->cur = csp->client_iob->buf;
4255       assert(csp->client_iob->eod == csp->client_iob->buf + bytes_to_shift + data_length);
4256       csp->client_iob->eod = csp->client_iob->buf + data_length;
4257       memset(csp->client_iob->eod, '\0', (size_t)bytes_to_shift);
4258
4259       csp->flags |= CSP_FLAG_PIPELINED_REQUEST_WAITING;
4260    }
4261    else
4262    {
4263       /*
4264        * We mainly care about resetting client_iob->cur so we don't
4265        * waste buffer space at the beginning and don't mess up the
4266        * request restoration done by cgi_show_request().
4267        *
4268        * Freeing the buffer itself isn't technically necessary,
4269        * but makes debugging more convenient.
4270        */
4271       clear_iob(csp->client_iob);
4272    }
4273 }
4274 #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */
4275
4276
4277 /*********************************************************************
4278  *
4279  * Function    :  serve
4280  *
4281  * Description :  This is little more than chat.  We only "serve" to
4282  *                to close (or remember) any socket that chat may have
4283  *                opened.
4284  *
4285  * Parameters  :
4286  *          1  :  csp = Current client state (buffers, headers, etc...)
4287  *
4288  * Returns     :  N/A
4289  *
4290  *********************************************************************/
4291 static void serve(struct client_state *csp)
4292 {
4293    int config_file_change_detected = 0; /* Only used for debugging */
4294 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
4295 #ifdef FEATURE_CONNECTION_SHARING
4296    static int monitor_thread_running = 0;
4297 #endif /* def FEATURE_CONNECTION_SHARING */
4298    int continue_chatting = 0;
4299
4300    log_error(LOG_LEVEL_CONNECT, "Accepted connection from %s on socket %d",
4301       csp->ip_addr_str, csp->cfd);
4302
4303    do
4304    {
4305       unsigned int latency;
4306
4307 #ifdef FEATURE_HTTPS_INSPECTION
4308       if (continue_chatting && client_use_ssl(csp))
4309       {
4310          continue_https_chat(csp);
4311       }
4312       else
4313 #endif
4314       {
4315          chat(csp);
4316       }
4317
4318       /*
4319        * If the request has been crunched,
4320        * the calculated latency is zero.
4321        */
4322       latency = (unsigned)(csp->server_connection.response_received -
4323          csp->server_connection.request_sent) / 2;
4324
4325       if ((csp->flags & CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE)
4326          && (csp->flags & CSP_FLAG_CRUNCHED)
4327          && (csp->expected_client_content_length != 0))
4328       {
4329          csp->flags |= CSP_FLAG_SERVER_SOCKET_TAINTED;
4330          log_error(LOG_LEVEL_CONNECT,
4331             "Tainting client socket %d due to unread data.", csp->cfd);
4332       }
4333
4334       continue_chatting = (csp->config->feature_flags
4335          & RUNTIME_FEATURE_CONNECTION_KEEP_ALIVE)
4336          && !(csp->flags & CSP_FLAG_SERVER_SOCKET_TAINTED)
4337          && (csp->cfd != JB_INVALID_SOCKET)
4338          && (csp->flags & CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE)
4339          && ((csp->flags & CSP_FLAG_SERVER_CONTENT_LENGTH_SET)
4340             || (csp->flags & CSP_FLAG_CHUNKED));
4341
4342       if (!(csp->flags & CSP_FLAG_CRUNCHED)
4343          && (csp->server_connection.sfd != JB_INVALID_SOCKET))
4344       {
4345          if (!(csp->flags & CSP_FLAG_SERVER_KEEP_ALIVE_TIMEOUT_SET))
4346          {
4347             csp->server_connection.keep_alive_timeout = csp->config->default_server_timeout;
4348          }
4349          if (!(csp->flags & CSP_FLAG_SERVER_CONNECTION_KEEP_ALIVE)
4350             || (csp->flags & CSP_FLAG_SERVER_SOCKET_TAINTED)
4351             || !socket_is_still_alive(csp->server_connection.sfd)
4352             || !(latency < csp->server_connection.keep_alive_timeout))
4353          {
4354             log_error(LOG_LEVEL_CONNECT,
4355                "Closing server socket %d connected to %s. "
4356                "Keep-alive: %u. Tainted: %u. Socket alive: %u. Timeout: %u.",
4357                csp->server_connection.sfd, csp->server_connection.host,
4358                0 != (csp->flags & CSP_FLAG_SERVER_CONNECTION_KEEP_ALIVE),
4359                0 != (csp->flags & CSP_FLAG_SERVER_SOCKET_TAINTED),
4360                socket_is_still_alive(csp->server_connection.sfd),
4361                csp->server_connection.keep_alive_timeout);
4362 #ifdef FEATURE_CONNECTION_SHARING
4363             if (csp->config->feature_flags & RUNTIME_FEATURE_CONNECTION_SHARING)
4364             {
4365                forget_connection(csp->server_connection.sfd);
4366             }
4367 #endif /* def FEATURE_CONNECTION_SHARING */
4368 #ifdef FEATURE_HTTPS_INSPECTION
4369             close_server_ssl_connection(csp);
4370 #endif
4371             close_socket(csp->server_connection.sfd);
4372             mark_connection_closed(&csp->server_connection);
4373          }
4374       }
4375
4376       if (continue_chatting && any_loaded_file_changed(csp))
4377       {
4378          continue_chatting = 0;
4379          config_file_change_detected = 1;
4380       }
4381 #ifdef FEATURE_HTTPS_INSPECTION
4382       if (continue_chatting && client_use_ssl(csp) &&
4383          csp->ssl_with_client_is_opened == 0)
4384       {
4385          continue_chatting = 0;
4386          log_error(LOG_LEVEL_CONNECT, "Client socket %d is no longer usable. "
4387             "The TLS session has been terminated.", csp->cfd);
4388       }
4389 #endif
4390
4391       if (continue_chatting)
4392       {
4393          if (((csp->flags & CSP_FLAG_PIPELINED_REQUEST_WAITING) != 0)
4394             && socket_is_still_alive(csp->cfd))
4395          {
4396             log_error(LOG_LEVEL_CONNECT, "Client request %d has been "
4397                "pipelined on socket %d and the socket is still alive.",
4398                csp->requests_received_total+1, csp->cfd);
4399             prepare_csp_for_next_request(csp);
4400             continue;
4401          }
4402
4403          if (0 != (csp->flags & CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE))
4404          {
4405             if (csp->server_connection.sfd != JB_INVALID_SOCKET)
4406             {
4407                log_error(LOG_LEVEL_CONNECT,
4408                   "Waiting for the next client request on socket %d. "
4409                   "Keeping the server socket %d to %s open.",
4410                   csp->cfd, csp->server_connection.sfd, csp->server_connection.host);
4411             }
4412             else
4413             {
4414                log_error(LOG_LEVEL_CONNECT,
4415                   "Waiting for the next client request on socket %d. "
4416                   "No server socket to keep open.", csp->cfd);
4417             }
4418          }
4419
4420          if ((csp->flags & CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE)
4421             && data_is_available(csp->cfd, (int)csp->config->keep_alive_timeout)
4422             && socket_is_still_alive(csp->cfd))
4423          {
4424             log_error(LOG_LEVEL_CONNECT,
4425                "Data arrived in time on client socket %d. Requests so far: %u",
4426                csp->cfd, csp->requests_received_total);
4427             prepare_csp_for_next_request(csp);
4428          }
4429          else
4430          {
4431 #ifdef FEATURE_CONNECTION_SHARING
4432             if ((csp->config->feature_flags & RUNTIME_FEATURE_CONNECTION_SHARING)
4433                && (csp->server_connection.sfd != JB_INVALID_SOCKET)
4434                && (socket_is_still_alive(csp->server_connection.sfd))
4435 #ifdef FEATURE_HTTPS_INSPECTION
4436                && !server_use_ssl(csp)
4437 #endif
4438                 )
4439             {
4440                time_t time_open = time(NULL) - csp->server_connection.timestamp;
4441
4442                if (csp->server_connection.keep_alive_timeout < time_open - (time_t)latency)
4443                {
4444                   break;
4445                }
4446
4447           &nb