Debian Codegraph @

src/HTTPHeader.cpp

raw
1 //ll support, instructions and copyright go to:
2 // http://e2guardian.org/
3 // Released under the GPL v2, with the OpenSSL exception described in the README file.
4
5 // INCLUDES
6
7 #ifdef HAVE_CONFIG_H
8 #include "e2config.h"
9 #endif
10 #include "HTTPHeader.hpp"
11 #include "Socket.hpp"
12 #include "OptionContainer.hpp"
13 #include "FDTunnel.hpp"
14 #include "Logger.hpp"
15
16 #include <unistd.h>
17 #include <sys/socket.h>
18 #include <exception>
19 #include <time.h>
20 #include <stdio.h>
21 #include <string.h>
22 #include <cerrno>
23 #include <zlib.h>
24
25 // GLOBALS
26 extern OptionContainer o;
27 extern thread_local std::string thread_id;
28
29 // regexp for decoding %xx in URLs
30 extern RegExp urldecode_re;
31
32 // IMPLEMENTATION
33
34 // set timeout for socket operations
35 void HTTPHeader::setTimeout(int t)
36 {
37 timeout = t;
38 }
39
40 // reset header object for future use
41 void HTTPHeader::reset()
42 {
43 if (dirty) {
44 header.clear();
45 waspersistent = false;
46 ispersistent = false;
47
48 cachedurl = "";
49
50 addheaderchecked = false;
51 isheaderadded = false;
52 searchwds = "";
53 issearch = false;
54 searchchecked = false;
55 clcached = false;
56 expects_100 = false;
57
58 mitm = false;
59 isdirect = false;
60 chunked = false;
61
62 phost = NULL;
63 pport = NULL;
64 pcontentlength = NULL;
65 pcontenttype = NULL;
66 pproxyauthorization = NULL;
67 pauthorization = NULL;
68 pproxyauthenticate = NULL;
69 pcontentdisposition = NULL;
70 puseragent = NULL;
71 pxforwardedfor = NULL;
72 pcontentencoding = NULL;
73 ptransfercoding = NULL;
74 ptransferencoding = NULL;
75 pproxyconnection = NULL;
76 pkeepalive = NULL;
77
78 useragent = "";
79 contenttype = "";
80 contentencoding = "";
81 transferencoding = "";
82
83 contentlength = -1;
84 requesttype = "";
85 returncode = 0;
86 first_line_sent = false;
87 header_all_sent = false;
88
89 dirty = false;
90
91 isProxyRequest = false;
92 icap = false;
93
94 delete postdata;
95 postdata = NULL;
96 postdata_len = 0;
97 }
98 }
99
100 // *
101 // *
102 // * header value and type checks
103 // *
104 // *
105
106 // grab request type (GET, HEAD etc.)
107 String HTTPHeader::requestType()
108 {
109 return requesttype;
110 }
111
112 // grab return code
113 int HTTPHeader::returnCode()
114 {
115 return returncode;
116 }
117
118 // grab content length
119 off_t HTTPHeader::contentLength()
120 {
121 return contentlength;
122 }
123
124 // grab the auth type
125 String HTTPHeader::getAuthType()
126 {
127 if (pproxyauthorization != NULL) {
128 return pproxyauthorization->after(" ").before(" ");
129 }
130 return "";
131 }
132
133 // check the request's return code to see if it's an auth required message
134 bool HTTPHeader::authRequired()
135 {
136 if (returncode == 407)
137 return true;
138 else
139 return false;
140 }
141
142 // grab content disposition
143 String HTTPHeader::disposition()
144 {
145 if (pcontentdisposition != NULL) {
146 String filename(pcontentdisposition->after("filename").after("="));
147 if (filename.contains(";"))
148 filename = filename.before(";");
149 filename.removeWhiteSpace(); // incase of trailing space
150 if (filename.contains("\"")) {
151 return filename.after("\"").before("\"");
152 }
153 return filename;
154 // example format:
155 // Content-Disposition: attachment; filename="filename.ext"
156 // Content-Disposition: attachment; filename=filename.ext
157 // Content-Disposition: filename="filename.ext"
158 // 3rd format encountered from download script on realVNC's
159 // website. notice it does not contain any semicolons! PRA 4-11-2005
160 }
161 return ""; // it finds the header proposed filename
162 }
163
164 // grab the user agent
165 String HTTPHeader::userAgent()
166 {
167 return useragent;
168 }
169
170 // grab the content type header
171 String HTTPHeader::getContentType()
172 {
173 if (pcontenttype != NULL) {
174 String mimetype(pcontenttype->after(" "));
175 if (mimetype.length() < 1)
176 return "-";
177
178 mimetype.chop();
179
180 if (mimetype.contains(" "))
181 mimetype = mimetype.before(" ");
182
183 if (mimetype.contains(";"))
184 mimetype = mimetype.before(";");
185
186 mimetype.toLower();
187 return mimetype;
188 }
189 return "-";
190 }
191
192 // grab the boundary for multi-part MIME POST data
193 String HTTPHeader::getMIMEBoundary()
194 {
195 if (pcontenttype != NULL) {
196 String boundary(pcontenttype->after(" "));
197 boundary.chop();
198
199 if (!boundary.contains("boundary="))
200 return "";
201
202 boundary = boundary.after("boundary=");
203 if (boundary.contains(";"))
204 boundary = boundary.after(";");
205
206 if (boundary[0] == '"') {
207 boundary.lop();
208 boundary.chop();
209 }
210
211 return boundary;
212 }
213 return "";
214 }
215
216 // does the given content type string match our headers?
217 bool HTTPHeader::isContentType(const String &t, FOptionContainer* &foc)
218 {
219 DEBUG_debug("mime type: ", getContentType());
220
221 // Do standard check first!
222 if (getContentType().startsWith(t))
223 return true;
224
225 // Only check text_mime types if ContentType request is 'text'
226 if (t == "text") {
227 String mime = getContentType();
228 std::deque<std::string> text_mime = foc->text_mime;
229 int size = (int) text_mime.size();
230 int i;
231 for (i = 0; i < size; i++) {
232 if (mime.startsWith(text_mime[i])) {
233 DEBUG_debug( "mimes match : ", text_mime[i]);
234 return true;
235 }
236 else {
237 DEBUG_debug("mimes check : ", text_mime[i]);
238 }
239 }
240 }
241 DEBUG_debug("mimes result : false !");
242 return false;
243 }
244
245 bool HTTPHeader::OKtoFilterMime(FOptionContainer* &foc)
246 {
247 String mime = getContentType();
248
249 // check stop list
250 std::deque<std::string> text_mime = foc->text_mime_stop;
251 int size = (int) text_mime.size();
252 int i;
253 for (i = 0; i < size; i++) {
254 if (mime.startsWith(text_mime[i])) {
255 DEBUG_debug( "mimes match stop: ", text_mime[i]);
256 return false;
257 }
258 else {
259 DEBUG_debug("mimes check stop: ", text_mime[i]);
260 }
261 }
262
263 //check if 'text'
264 if(isContentType("text",foc) || isContentType("-",foc)) return true;
265 return false;
266 }
267
268 // grab contents of X-Forwarded-For header
269 // Modification based on a submitted patch by
270 // Jimmy Myrick ([email protected])
271 std::string HTTPHeader::getXForwardedForIP()
272 {
273 if (pxforwardedfor != NULL) {
274 String line(pxforwardedfor->after(": "));
275 line.chop();
276 return std::string(line.toCharArray());
277 }
278 return "";
279 }
280
281 // check the return code to see if it's a redirection request
282 bool HTTPHeader::isRedirection()
283 {
284 // The 1st line of the header for a redirection is thus:
285 // HTTP/1.(0|1) 3xx
286 if (header.size() < 1) {
287 return false;
288 } // sometimes get called b 4 read
289 String answer(header.front().after(" ").before(" "));
290 if (answer[0] == '3' && answer.length() == 3) {
291 return true;
292 }
293 return false;
294 }
295
296 // grab the contents of Proxy-Authorization header
297 // returns base64-decoding of the chunk of data after the auth type string
298 std::string HTTPHeader::getAuthData()
299 {
300 if (pproxyauthorization != NULL) {
301 String line(pproxyauthorization->after(" ").after(" "));
302 //line.chop();
303 DEBUG_auth(("pauth string size:"),line.length()," ", line);
304 String ret = decodeb64(line);
305 DEBUG_auth(("pauth after decodeb64:"), ret);
306 ret.removeChar('\0');
307 DEBUG_auth(("returning:"), ret);
308 return ret; // it's base64 MIME encoded
309 }
310 return "";
311 }
312
313 std::string HTTPHeader::getAuthHeader()
314 {
315 if (pheaderident != NULL) {
316 String line (pheaderident->after(" "));
317 line.resize(line.length() - 1);
318 return line;
319 }
320 return "";
321 }
322
323 // grab raw contents of Proxy-Authorization header without decoding
324 std::string HTTPHeader::getRawAuthData()
325 {
326 if (pproxyauthorization != NULL) {
327 return pproxyauthorization->after(" ").after(" ");
328 }
329 return "";
330 }
331
332 // do we have a non-identity content encoding? this means body is compressed
333 bool HTTPHeader::isCompressed()
334 {
335 if (pcontentencoding != NULL) {
336 if (pcontentencoding->indexOf("identity") != -1) {
337 // http says this
338 // should not be here, but not must not
339 return false;
340 }
341 DEBUG_debug( "is compressed!");
342 return true; // i.e. encoded with something other than clear
343 }
344 return false;
345 }
346
347 // grab content encoding header
348 String HTTPHeader::contentEncoding()
349 {
350 if (pcontentencoding != NULL) {
351 String ce(pcontentencoding->after(": "));
352 ce.toLower();
353 return ce;
354 }
355 return ""; // we need a default don't we?
356 }
357
358 // grab transfer encoding header
359 String HTTPHeader::transferEncoding()
360 {
361 if (ptransferencoding != NULL) {
362 String ce(ptransferencoding->after(": "));
363 ce.toLower();
364 return ce;
365 }
366 return ""; // we need a default don't we?
367 }
368
369 // *
370 // *
371 // * header modifications
372 // *
373 // *
374
375 // squid adds this so if more support it it may be useful one day
376 void HTTPHeader::addXForwardedFor(const std::string &clientip)
377 {
378 if (!isdirect) {
379 std::string line("X-Forwarded-For: " + clientip + "\r");
380 header.push_back(String(line.c_str()));
381 }
382 }
383
384 // set content length header to report given lenth
385 void HTTPHeader::setContentLength(int newlen)
386 {
387 if (pcontentlength != NULL) {
388 (*pcontentlength) = "Content-Length: " + String(newlen) + "\r";
389 }
390 contentlength = newlen;
391 clcached = true;
392 }
393
394 // set the proxy-connection header to allow persistence (or not)
395 void HTTPHeader::makePersistent(bool persist)
396 {
397 if (persist) {
398 // Only make persistent if it originally was, but now isn't.
399 // The intention isn't to change browser behaviour, just to
400 // un-do any connection downgrading which E2 may have performed
401 // earlier.
402 if (waspersistent && !ispersistent) {
403 if (pproxyconnection != NULL) {
404 (*pproxyconnection) = pproxyconnection->before(":") + ": keep-alive\r";
405 } else {
406 header.push_back(String("Connection: keep-alive\r"));
407 pproxyconnection = &(header.back());
408 }
409 ispersistent = true;
410 }
411 } else {
412 // Only downgrade to non-persistent if it isn't currently persistent.
413 if (ispersistent) {
414 if (pproxyconnection != NULL) {
415 (*pproxyconnection) = pproxyconnection->before(":") + ": close\r";
416 } else {
417 header.push_back(String("Connection: close\r"));
418 pproxyconnection = &(header.back());
419 }
420 ispersistent = false;
421 }
422 }
423 }
424
425 // make the request look like it's come from/to the origin server
426 void HTTPHeader::makeTransparent(bool incoming)
427 {
428 DEBUG_trace("");
429 if (incoming) {
430 // remove references to the proxy before sending to browser
431 if (pproxyconnection != NULL) {
432 String temp = pproxyconnection->after(":");
433 (*pproxyconnection) = "Connection:";
434 (*pproxyconnection) += temp;
435 }
436 if (pproxyauthenticate != NULL) {
437 String temp = pproxyauthenticate->after(":");
438 (*pproxyauthenticate) = "WWW-Authenticate:";
439 (*pproxyauthenticate) += temp;
440 }
441 if (returnCode() == 407) {
442 String temp = header.front().before(" ");
443 String temp2 = header.front().after(" ").after(" ");
444 header.front() = temp + " 401 ";
445 header.front() += temp2;
446 }
447 } else {
448 // remove references to origin server before sending to proxy
449 if (pauthorization != NULL) {
450 String temp = pauthorization->after(":");
451 (*pauthorization) = "Proxy-Authorization:";
452 (*pauthorization) += temp;
453 pproxyauthorization = pauthorization;
454 pauthorization = NULL;
455 }
456 if (pproxyconnection != NULL) {
457 String temp = pproxyconnection->after(":");
458 (*pproxyconnection) = "Connection:";
459 (*pproxyconnection) += temp;
460 }
461 // call this to fudge the URL into something Squid likes
462 getUrl();
463 }
464 }
465
466 // return a modified accept-encoding header, based on the one supplied,
467 // but with "identity" added and only supported encodings allowed.
468 String HTTPHeader::modifyEncodings(String e)
469 {
470
471 // There are 4 types of encoding: gzip, deflate, compress and identity
472 // deflate is in zlib format
473 // compress is in unix compress format
474 // identity is uncompressed and supported by all browsers (obviously)
475 // we do not support compress
476
477 e.toLower();
478 String o("Accept-Encoding: identity");
479 #if ZLIB_VERNUM < 0x1210
480 #warning 'Accept-Encoding: gzip' is disabled
481 #else
482 if (e.contains("gzip")) {
483 o += ",gzip";
484 }
485 #endif
486 if (e.contains("deflate")) {
487 o += ",deflate";
488 }
489
490 if (e.contains("pack200-gzip")) {
491 o += ",pack200-gzip";
492 }
493
494 return o;
495 }
496
497 // set content length to report the given length, and strip content encoding
498 void HTTPHeader::removeEncoding(int newlen)
499 {
500 if (pcontentlength != NULL) {
501 (*pcontentlength) = "Content-Length: " + String(newlen) + "\r";
502 }
503 // this may all be overkill. since we strip everything out of the outgoing
504 // accept-encoding header that we don't support, we won't be getting anything
505 // back again that we don't support, in theory. leave new code commented
506 // unless it proves to be necessary further down the line. PRA 20-10-2005
507 if (pcontentencoding != NULL) {
508 /*
509 DEBUG_debug("Stripping Content-Encoding header");
510 DEBUG_debug("Old: ", header[i]);
511 // only strip supported compression types
512 String temp(header[i].after(":"));
513 temp.removeWhiteSpace();
514 String newheader;
515 // iterate over comma-separated list of encodings
516 while (temp.length() != 0) {
517 if (!(temp.startsWith("gzip") || temp.startsWith("deflate"))) {
518 // add other, unstripped encoding types back into the header
519 if (newheader.length() != 0)
520 newheader += ", ";
521 newheader += (temp.before(",").length() != 0 ? temp.before(",") : temp);
522 }
523 temp = temp.after(",");
524 temp.removeWhiteSpace();
525 }
526 if (newheader.length() == 0)*/
527 (*pcontentencoding) = "X-DansGuardian-Removed: Content-Encoding\r";
528 /* else
529 header[i] = "Content-Encoding: "+newheader;
530 DEBUG_debug("New: ", header[i]);
531 */
532 }
533 }
534
535 void HTTPHeader::setConnect(String &con_site) {
536 if (requestType() != "CONNECT") return;
537 header.front() = header.front().before(" ") + " " + con_site + ":" + String(port) + " " + header.front().after(" ").after(" ");
538 //remove all other headers
539 if (header.size() > 1) {
540 header.erase(header.begin()+1, header.end());
541 }
542 if (phost != NULL) {
543 phost = nullptr;
544 }
545 }
546
547 // modifies the URL in all relevant header lines after a regexp search and replace
548 // setURL Code originally from from Ton Gorter 2004
549 void HTTPHeader::setURL(String &url)
550 {
551 String hostname;
552 bool https = (url.before("://") == "https");
553 if(requestType() == "CONNECT"){
554 https = true;
555 }
556 int port = (https ? 443 : 80);
557
558 if (!url.after("://").contains("/")) {
559 url += "/";
560 }
561 hostname = url.after("://").before("/");
562 if (hostname.contains("@")) { // Contains a username:password combo
563 hostname = hostname.after("@");
564 }
565 if (hostname.contains(":")) {
566 port = hostname.after(":").toInteger();
567 if (port == 0 || port > 65535) {
568 port = (https ? 443 : 80);
569 }
570 hostname = hostname.before(":"); // chop off the port bit
571 }
572
573 DEBUG_debug("setURL: header.front() changed from: ", header.front());
574 if (https && header.front().startsWith("CONNECT"))
575 // Should take form of "CONNECT example.com:443 HTTP/1.0" for SSL
576 header.front() = header.front().before(" ") + " " + hostname + ":" + String(port) + " " + header.front().after(" ").after(" ");
577 else
578 header.front() = header.front().before(" ") + " " + url + " " + header.front().after(" ").after(" ");
579
580 DEBUG_debug(" to: ", header.front());
581
582 if (phost != NULL) {
583 DEBUG_debug("setURL: header[] line changed from: ", (*phost));
584 (*phost) = String("Host: ") + hostname;
585 if (port != (https ? 443 : 80)) {
586 (*phost) += ":";
587 (*phost) += String(port);
588 }
589 (*phost) += "\r";
590 DEBUG_debug(" to ", (*phost));
591 }
592 if (pport != NULL) {
593 DEBUG_debug("setURL: header[] line changed from: ", (*pport));
594 (*pport) = String("Port: ") + String(port) + "\r";
595 DEBUG_debug(" to ", (*pport));
596 }
597 // Don't just cache the URL we're sent - getUrl() performs some other
598 // processing, notably stripping the port part. Caching here will
599 // bypass all that.
600 //cachedurl = url.toCharArray();
601 cachedurl = ""; // blank cachedurl so that getUrl will re-generate it
602 }
603
604 // Does a regexp search and replace.
605 // urlRegExp Code originally from from Ton Gorter 2004
606 bool HTTPHeader::regExp(String &line, std::deque<RegExp> &regexp_list, std::deque<String> &replacement_list)
607 {
608 RegExp *re;
609 RegResult Rre;
610 String replacement;
611 String repstr;
612 String newLine;
613 bool linemodified = false;
614 unsigned int i;
615 unsigned int j, k;
616 unsigned int s = regexp_list.size();
617 unsigned int matches, submatches;
618 unsigned int match;
619 unsigned int srcoff;
620 unsigned int nextoffset;
621 unsigned int matchlen;
622 unsigned int oldlinelen;
623
624 if ( (line.empty()) || line.length() < 3)
625 return false;
626
627 // iterate over our list of precompiled regexes
628 for (i = 0; i < s; i++) {
629 newLine = "";
630 re = &(regexp_list[i]);
631 if (re->match(line.toCharArray(), Rre)) {
632 repstr = replacement_list[i];
633 matches = Rre.numberOfMatches();
634
635 srcoff = 0;
636
637 for (j = 0; j < matches; j++) {
638 nextoffset = Rre.offset(j);
639 matchlen = Rre.length(j);
640
641 // copy next chunk of unmodified data
642 if (nextoffset > srcoff) {
643 newLine += line.subString(srcoff, nextoffset - srcoff);
644 srcoff = nextoffset;
645 }
646
647 // Count number of submatches (brackets) in replacement string
648 for (submatches = 0; j + submatches + 1 < matches; submatches++)
649 if (Rre.offset(j + submatches + 1) + Rre.length(j + submatches + 1) > srcoff + matchlen)
650 break;
651
652 // \1 and $1 replacement
653 replacement = "";
654 for (k = 0; k < repstr.length(); k++) {
655 // find \1..\9 and $1..$9 and fill them in with submatched strings
656 if ((repstr[k] == '\\' || repstr[k] == '$') && repstr[k + 1] >= '1' && repstr[k + 1] <= '9') {
657 match = repstr[++k] - '0';
658 if (match <= submatches) {
659 replacement += Rre.result(j + match).c_str();
660 }
661 } else {
662 // unescape \\ and \$, and add non-backreference characters to string
663 if (repstr[k] == '\\' && (repstr[k + 1] == '\\' || repstr[k + 1] == '$'))
664 k++;
665 replacement += repstr.subString(k, 1);
666 }
667 }
668
669 // copy filled in replacement string
670 newLine += replacement;
671 srcoff += matchlen;
672 j += submatches;
673 }
674 oldlinelen = line.length();
675 if (srcoff < oldlinelen) {
676 newLine += line.subString(srcoff, oldlinelen - srcoff);
677 }
678 DEBUG_debug("Line modified! (", line, " -> ", newLine, ")" );
679 // copy newLine into line and continue with other regexes
680 line = newLine;
681 linemodified = true;
682 }
683 }
684
685 return linemodified;
686 }
687
688
689
690 String HTTPHeader::redirecturl()
691 {
692 return redirect;
693 }
694
695 bool HTTPHeader::addHeader(String &newheader) {
696 if (newheader.size() > 0) {
697 isheaderadded = true;
698 addheaderchecked = true;
699 std::string line(newheader + "\r");
700 header.push_back(String(line.c_str()));
701 DEBUG_debug("addheader = ", newheader);
702 return true;
703 }
704 return false;
705 }
706
707
708 void HTTPHeader::setPostData(const char *data, size_t len)
709 {
710 if(postdata != nullptr) delete postdata;
711 postdata = new char[len];
712 memcpy(postdata, data, len);
713 postdata_len = len;
714 }
715
716 // *
717 // *
718 // * detailed header checks & fixes
719 // *
720 // *
721
722 // is a URL malformed?
723 bool HTTPHeader::malformedURL(const String &url)
724 {
725 String host(url.after("://"));
726 if (host.contains("/"))
727 host = host.before("/");
728 if (host.length() < 2) {
729 DEBUG_debug("host len too small!");
730 return true;
731 }
732 if (host.contains(":"))
733 host = host.before(":");
734 // endsWith . check removed as this format is used by Apple Configurator Updates
735 // if (host.contains("..") || host.endsWith(".")) {
736 if (host.contains("..")) {
737 DEBUG_debug("double dots in domain name!");
738 return true;
739 }
740 int i, len;
741 unsigned char c;
742 len = host.length();
743 bool containsletter = false;
744 for (i = 0; i < len; i++) {
745 c = (unsigned char)host[i];
746 // If it contains something other than numbers, dots, or [a-fx] (hex encoded IPs),
747 // IP obfuscation can be ruled out.
748 if (!containsletter && (((c < '0') || (c > '9'))
749 && (c != '.') && (c != 'x') && (c != 'X')
750 && ((c < 'a') || (c > 'f'))
751 && ((c < 'A') || (c > 'F'))))
752 containsletter = true;
753 if (!(c >= 'a' && c <= 'z') && !(c >= 'A' && c <= 'Z')
754 && !(c >= '0' && c <= '9') && c != '.' && c != '-' && c != '_') {
755 // only allowed letters, digits, hiphen, dots
756 DEBUG_debug("bad char in hostname!");
757 return true;
758 }
759 }
760 // no IP obfuscation going on
761 if (containsletter)
762 return false;
763 else
764 DEBUG_debug("Checking for IP obfuscation in ", host);
765
766 // Check no IP obfuscation is going on
767 // This includes IPs encoded as a single decimal number,
768 // fully or partly hex encoded, and octal encoded
769 bool first = true;
770 bool obfuscation = false;
771 if (host.endsWith("."))
772 host.chop();
773 do {
774 if (!first)
775 host = host.after(".");
776 first = false;
777 String hostpart(host);
778 if (host.contains("."))
779 hostpart = hostpart.before(".");
780 // If any part of the host starts with a letter, any letter,
781 // then we must have a hostname rather than an IP (obscured
782 // or otherwise). TLDs never start with a number.
783 if ((hostpart[0] >= 'a' && hostpart[0] <= 'z') || (hostpart[0] >= 'A' && hostpart[0] <= 'Z'))
784 return false;
785 // If any part of the host begins with 0, it may be hex or octal
786 if ((hostpart[0] == '0') && (hostpart.length() > 1)) {
787 obfuscation = true;
788 continue;
789 }
790 // Also check range, for decimal obfuscation.
791 int part = hostpart.toInteger();
792 if ((part < 0) || (part > 255))
793 obfuscation = true;
794 } while (host.contains("."));
795 // If we have any obfuscated parts, and haven't proven it's a hostname, it's invalid.
796 return obfuscation;
797 }
798
799
800 // Show headers values for debug purpose
801
802 void HTTPHeader::dbshowheader(String *url, const char *clientip)
803 {
804 return; // temporally disable
805 std::string reqres, inout;
806 if (is_response) {
807 reqres = "RES";
808 } else {
809 reqres = "REQ";
810 }
811 //const char *tid = thread_id.c_str();
812
813 if (header.size() != 0){
814 String *line;
815 DEBUG_debug("Client: START ", reqres, "-------------------------------" );
816 for (std::deque<String>::iterator i = header.begin(); i != header.end(); i++) {
817 line = &(*i);
818 String line2 = *line;
819 if (header.front().startsWith("HT")) {
820 inout = "IN";
821 } else {
822 inout = "OUT";
823 }
824 DEBUG_debug(inout, ": Client IP ", clientip, " ", line2 );
825 }
826 DEBUG_debug("Client: END ", reqres, " -------------------------------");
827 } else {
828 DEBUG_debug("Call : from HTTPHeader.cpp to dbshowheader but", reqres, " header is empty" );
829 }
830 }
831
832 void HTTPHeader::dbshowheader(bool outgoing)
833 {
834 return; // temporally disable
835 std::string reqres, inout;
836 if (is_response) {
837 reqres = "RES";
838 } else {
839 reqres = "REQ";
840 }
841 if (outgoing)
842 inout = reqres + "OUT";
843 else
844 inout = reqres + "IN";
845 // std::hash<std::thread::id> htid;
846 // std::thread::id tid = std::this_thread::get_id();
847
848 if (header.size() != 0){
849 String *line;
850 DEBUG_debug("Client: START-------------------------------");
851 for (std::deque<String>::iterator i = header.begin(); i != header.end(); i++) {
852 line = &(*i);
853 String line2 = *line;
854 DEBUG_debug(inout,": dbshowheader bool: ", line2);
855 }
856 DEBUG_debug("Client: END-------------------------------");
857 } else {
858 DEBUG_debug("Call : from HTTPHeader.cpp to dbshowheader but header is empty");
859 }
860 }
861
862
863
864 // fix bugs in certain web servers that don't obey standards.
865 // actually, it's us that don't obey standards - HTTP RFC says header names
866 // are case-insensitive. - Anonymous SF Poster, 2006-02-23
867 void HTTPHeader::checkheader(bool allowpersistent)
868 {
869 bool outgoing = !is_response;
870 // if (header.front().startsWith("HT")) {
871 // outgoing = false;
872 // }
873 String tp;
874
875 if (header.size() > 1) {
876 for (std::deque<String>::iterator i = header.begin() + 1; i != header.end(); i++) { // check each line in the headers
877 // index headers - try to perform the checks in the order the average browser sends the headers.
878 // also only do the necessary checks for the header type (sent/received).
879 // Sequencial if else
880 if (outgoing && (phost == NULL) && i->startsWithLower("host:")) {
881 phost = &(*i);
882 // don't allow through multiple host headers
883 } else if (outgoing && (phost != NULL) && i->startsWithLower("host:")) {
884 i->assign("X-E2G-IgnoreMe: removed multiple host headers\r");
885 } else if (outgoing && (puseragent == NULL) && i->startsWithLower("user-agent:")) {
886 puseragent = &(*i);
887 useragent = *i;
888 if(!useragent.headerVal())
889 puseragent = NULL;
890 } else if (outgoing && i->startsWithLower("accept-encoding:")) {
891 (*i) = "Accept-Encoding:" + i->after(":");
892 (*i) = modifyEncodings(*i);
893 (*i) += "\r";
894 } else if ((!outgoing) && (pcontentencoding == NULL) && i->startsWithLower("content-encoding:")) {
895 pcontentencoding = &(*i);
896 } else if ((!outgoing) && (pkeepalive == NULL) && i->startsWithLower("keep-alive:")) {
897 pkeepalive = &(*i);
898 } else if ((pcontenttype == NULL) && i->startsWithLower("content-type:")) {
899 pcontenttype = &(*i);
900 } else if ((pcontentlength == NULL) && i->startsWithLower("content-length:")) {
901 pcontentlength = &(*i);
902 tp = *i;
903 DEBUG_debug("tp =", tp);
904
905 if(!tp.headerVal())
906 pcontentlength = NULL;
907 else {
908 contentlength = tp.toOffset();
909 }
910 DEBUG_debug("tp =", tp, " Contentlen.int =", contentlength);
911
912 }
913 // is this ever sent outgoing?
914 else if ((pcontentdisposition == NULL) && i->startsWithLower("content-disposition:")) {
915 pcontentdisposition = &(*i);
916 } else if ((pproxyauthorization == NULL) && i->startsWithLower("proxy-authorization:")) {
917 pproxyauthorization = &(*i);
918 } else if ((pauthorization = NULL) && i->startsWithLower("authorization:")) {
919 pauthorization = &(*i);
920 } else if ((pproxyauthenticate == NULL) && i->startsWithLower("proxy-authenticate:")) {
921 pproxyauthenticate = &(*i);
922 } else if ((pproxyconnection == NULL) && (i->startsWithLower("proxy-connection:") || i->startsWithLower("connection:"))) {
923 pproxyconnection = &(*i);
924 } else if (outgoing && (pxforwardedfor == NULL) && i->startsWithLower("x-forwarded-for:")) {
925 pxforwardedfor = &(*i);
926 }
927 // this one's non-standard, so check for it last
928 else if (outgoing && (pport == NULL) && i->startsWithLower("port:")) {
929 pport = &(*i);
930 } else if ( i->startsWithLower("transfer-encoding:")) {
931 ptransferencoding = &(*i);
932 if (i->contains("chunked"))
933 chunked = true;
934 } else if ( i->startsWithLower("transfer-coding:")) {
935 ptransfercoding = &(*i);
936 } else if ( i->startsWithLower("expect: 100-continue")) {
937 expects_100 = true;
938 }
939
940 //Can be placed anywhere ..
941 if (outgoing && i->startsWithLower("upgrade-insecure-requests:")) {
942 i->assign("X-E2G-IgnoreMe: removed upgrade-insecure-requests\r");
943 }
944
945 // if ((o.log_header_value.size() != 0) && outgoing && (plogheadervalue == NULL) && i->startsWithLower(o.log_header_value)) {
946 // plogheadervalue = &(*i);
947 // }
948 if ((o.header.ident_header_value.size() != 0) && outgoing && (pheaderident == NULL) && i->startsWithLower(o.header.ident_header_value)) {
949 pheaderident = &(*i);
950 }
951
952 DEBUG_debug("Header value from client: ", *i);
953 }
954 }
955 //checkfirstheaderline(allowpersistent);
956 if (outgoing && !requesttype.startsWith("P")) // is not POST or PUT no body is allowed
957 {
958 DEBUG_debug("zero contentlength on request due to not POST/PUT ");
959 contentlength = 0;
960 }
961
962 if (is_response && ((returncode < 200) || (returncode == 204) || (returncode == 304))) // no content body allowed
963 {
964 DEBUG_debug("zero contentlength on response due to returncode ", String(returncode) );
965 contentlength = 0;
966 }
967
968 //work out if we should explicitly close this connection after this request
969 bool connectionclose = false;
970 if(!icap) {
971 if (pproxyconnection != NULL) {
972 if (pproxyconnection->contains("lose")) {
973 DEBUG_debug("CheckHeader: P-C says close");
974 connectionclose = true;
975 } else {
976 connectionclose = false;
977 }
978 } else {
979 connectionclose = true;
980 }
981 }
982
983 // Do not allow persistent connections on CONNECT requests - the browser thinks it has a tunnel
984 // directly to the external server, not a connection to the proxy, so it won't be re-used in the
985 // manner expected by E2 and will result in waiting for time-outs. Bug identified by Jason Deasi.
986 bool isconnect = false;
987 if (outgoing && header.front()[0] == 'C') {
988 DEBUG_debug("CheckHeader: CONNECT request detected");
989 isconnect = true;
990 }
991
992 DEBUG_debug("CheckHeader flags before normalisation: ",
993 " AP=", String(allowpersistent),
994 " PPC=", String(pproxyconnection != NULL),
995 " 1.1=", String(onepointone),
996 " connectionclose=", String(connectionclose),
997 " CL=", String(pcontentlength != NULL) );
998
999 if (connectionclose || (!onepointone && (outgoing ? isconnect : (pcontentlength == NULL)))) {
1000 // couldnt have done persistency even if we wanted to
1001 allowpersistent = false;
1002 }
1003
1004 if(!icap) {
1005 if (outgoing) {
1006 // Even though persistent CONNECT requests usually break things, waspersistent should
1007 // reflect the intention of the original request headers, or NTLM breaks.
1008 if (isconnect && !connectionclose) {
1009 waspersistent = true;
1010 }
1011 } else {
1012 if (!connectionclose && !(pcontentlength == NULL)) {
1013 waspersistent = true;
1014 }
1015 }
1016 }
1017
1018 DEBUG_debug("CheckHeader flags after normalisation: ",
1019 " AP=", String(allowpersistent),
1020 " WP=", String(waspersistent) );
1021
1022
1023 if(!icap) {
1024 // force the headers to reflect whether or not persistency is allowed
1025 // (modify pproxyconnection or add connection close/keep-alive - Client version, of course)
1026 if (allowpersistent ) {
1027 if (pproxyconnection == NULL) {
1028 DEBUG_debug("CheckHeader: Adding our own Proxy-Connection: Keep-Alive");
1029 header.push_back("Connection: keep-alive\r");
1030 pproxyconnection = &(header.back());
1031 } else {
1032 (*pproxyconnection) = "Connection: keep-alive\r";
1033 }
1034 } else {
1035 if (pproxyconnection == NULL) {
1036 DEBUG_debug("CheckHeader: Adding our own Proxy-Connection: Close");
1037 header.push_back("Connection: close\r");
1038 pproxyconnection = &(header.back());
1039 } else {
1040 (*pproxyconnection) = "Connection: close\r";
1041 }
1042 }
1043 }
1044
1045 ispersistent = allowpersistent;
1046
1047 // Normalise request headers (fix host, port, first line of header, etc. to all be consistent)
1048 if (outgoing) {
1049 String newurl(getUrl(true));
1050 setURL(newurl);
1051 }
1052 }
1053
1054 bool HTTPHeader::checkfirstheaderline() {
1055
1056 if(!header.front().contains("HTTP/")) {
1057 DEBUG_debug("CheckfirstHeader: not HTTP!");
1058 return false;
1059 }
1060 //if its http1.1
1061 onepointone = false;
1062 if (header.front().after("HTTP/").startsWith("1.1")) {
1063 DEBUG_debug("CheckHeader: HTTP/1.1 detected");
1064 onepointone = true;
1065 }
1066
1067 if (!is_response) { // i.e. is request - set request Type
1068 requesttype = header.front().before(" ");
1069 if(header.front().after(" ").startsWith("/"))
1070 isProxyRequest = false;
1071 else
1072 isProxyRequest = true;
1073 } else { // is response - set status code
1074 String tp = header.front().after(" ").before(" ");
1075 tp.removeWhiteSpace();
1076 returncode = tp.toInteger();
1077 if (returncode < 100 || returncode == 102) {
1078 DEBUG_debug("responsecode not valid ",returncode);
1079 returncode = 0;
1080 return false;
1081 }
1082 }
1083 return true;
1084 }
1085
1086 String HTTPHeader::getLogUrl(bool withport, bool isssl)
1087 {
1088
1089 String answer = getUrl(withport, isssl);
1090 if (mitm || isssl) {
1091 answer = "https://" + answer.after("://");
1092 }
1093 return answer;
1094 }
1095
1096 String HTTPHeader::getUrl(bool withport, bool isssl)
1097 {
1098 // Version of URL *with* port is not cached,
1099 // as vast majority of our code doesn't like
1100 // port numbers in URLs.
1101 if (cachedurl.length() > 0 && !withport)
1102 return cachedurl;
1103 port = 80;
1104 bool https = false;
1105 if (isssl) {
1106 mitm = isssl;
1107 isdirect = true;
1108 }
1109
1110 if ( header.size() == 0)
1111 {
1112 String ans;
1113 return ans;
1114 }
1115
1116 String hostname;
1117 String userpassword;
1118 String answer(header.front().after(" "));
1119 answer.removeMultiChar(' ');
1120 if (answer.after(" ").startsWith("HTTP/")) {
1121 answer = answer.before(" HTTP/");
1122 } else {
1123 answer = answer.before(" http/"); // just in case!
1124 }
1125 if (requestType() == "CONNECT") {
1126 https = true;
1127 port = 443;
1128 if (!answer.startsWith("https://")) {
1129 answer = "https://" + answer;
1130 }
1131 }
1132 if (pport != NULL) {
1133 port = pport->after(" ").toInteger();
1134 if (port == 0 || port > 65535)
1135 port = (https ? 443 : 80);
1136 }
1137 if (answer.length()) {
1138 if (answer[0] == '/') { // must be the latter above
1139 if (phost != NULL) {
1140 hostname = phost->after(" ");
1141 hostname.removeWhiteSpace();
1142 if (hostname.contains(":")) {
1143 port = hostname.after(":").toInteger();
1144 if (port == 0 || port > 65535) {
1145 port = (https ? 443 : 80);
1146 }
1147 hostname = hostname.before(":");
1148 }
1149 while (hostname.endsWith("."))
1150 hostname.chop();
1151 if (withport && (port != (https ? 443 : 80)))
1152 hostname += ":" + String(port);
1153 hostname = "http://" + hostname;
1154 answer = hostname + answer;
1155 }
1156 // Squid doesn't like requests in this format. Work around the fact.
1157 header.front() = requestType() + " " + answer + " HTTP/" + header.front().after(" HTTP/");
1158 } else { // must be in the form GET http://foo.bar:80/ HTML/1.0
1159 if (!answer.after("://").contains("/")) {
1160 answer += "/"; // needed later on so correct host is extracted
1161 }
1162 String protocol(answer.before("://"));
1163 hostname = answer.after("://");
1164 String url(hostname.after("/"));
1165 url.removeWhiteSpace(); // remove rubbish like ^M and blanks
1166 if (hostname.endsWith(".")) {
1167 hostname.chop();
1168 }
1169 if (url.length() > 0) {
1170 url = "/" + url;
1171 }
1172 hostname = hostname.before("/"); // extra / was added 4 here
1173 if (hostname.contains("@")) { // Contains a username:password combo
1174 userpassword = hostname.before("@");
1175 hostname = hostname.after("@");
1176 }
1177 if (hostname.contains(":")) {
1178 port = hostname.after(":").toInteger();
1179 if (port == 0 || port > 65535) {
1180 port = (https ? 443 : 80);
1181 }
1182 hostname = hostname.before(":"); // chop off the port bit
1183 }
1184 while (hostname.endsWith("."))
1185 hostname.chop();
1186 if (withport && (port != (https ? 443 : 80)))
1187 hostname += ":" + String(port);
1188 if (userpassword.length())
1189 answer = protocol + "://" + userpassword + "@" + hostname + url;
1190 else
1191 answer = protocol + "://" + hostname + url;
1192 }
1193 }
1194 // Some sites do now have urls ending with '//' and will not answer to just '/'
1195 // if (answer.endsWith("//")) {
1196 // answer.chop();
1197 // }
1198
1199 DEBUG_debug("from header url:", answer);
1200 // Don't include port numbers in the URL in the cached version.
1201 // Most of the code only copes with URLs *without* port specifiers.
1202 if (!withport)
1203 cachedurl = answer.toCharArray();
1204 return answer;
1205 }
1206
1207 String HTTPHeader::url()
1208 {
1209 return getUrl();
1210 }
1211
1212 // *
1213 // *
1214 // * Bypass URL/Cookie funcs
1215 // *
1216 // *
1217 // create a temporary bypass URL for the banned page
1218 String HTTPHeader::hashedURL(String *url, std::string *clientip,
1219 bool infectionbypass, std::string *user, FOptionContainer &fdl, bool fakecgi) {
1220 // filter/virus bypass hashes last for a certain time only
1221 String timecode(time(NULL) + (infectionbypass ? fdl.infection_bypass_mode
1222 : fdl.bypass_mode));
1223 // use the standard key in normal bypass mode, and the infection key in infection bypass mode
1224 String magic(infectionbypass ? fdl.imagic.c_str() : fdl.magic.c_str());
1225 magic += clientip->c_str();
1226 if(fdl.bypass_v2)
1227 magic += user->c_str();
1228 magic += timecode;
1229 String res(infectionbypass ? "GIBYPASS=" : "GBYPASS=");
1230 if (fakecgi) // it is an oversized file bypass
1231 res = "GOSBYPASS=";
1232 String hash;
1233 if (!url->after("://").contains("/")) {
1234 String newurl((*url));
1235 newurl += "/";
1236 hash = newurl.md5(magic.toCharArray());
1237 } else {
1238 hash = url->md5(magic.toCharArray());
1239 }
1240 if (fakecgi && fdl.cgi_bypass) //used for oversized files in fancy download manager
1241 hash = hash.md5(fdl.cgi_magic.c_str());
1242 res += hash;
1243 res += timecode;
1244 DEBUG_debug(" -generate Bypass hashedurl data ", clientip, " ", *url, " ", user, " ", timecode, " result ", res);
1245 return res;
1246 }
1247
1248 // create temporary bypass cookie
1249 String HTTPHeader::hashedCookie(String *url, const char *magic, std::string *clientip, int bypasstimestamp, std::string user) {
1250 String timecode(bypasstimestamp);
1251 String data(magic);
1252 data += clientip->c_str();
1253 // if(ldl->fg[filtergroup]->bypass_v2)
1254 data += user;
1255 data += timecode;
1256 DEBUG_debug(" -generate Bypass hashedCookie data ", clientip, " ", *url, " ", user, " ", timecode);
1257 String res(url->md5(data.toCharArray()));
1258 res += timecode;
1259
1260 DEBUG_debug(" -Bypass hashedCookie=", res );
1261 return res;
1262 }
1263
1264
1265 // chop the GBYPASS or GIBYPASS variable out of a bypass URL
1266 // This function ASSUMES that you really know what you are doing
1267 // Do NOT run this function unless you know that the URL contains a valid bypass code
1268 // Ernest W Lessenger
1269 //void HTTPHeader::chopBypass(String url, bool infectionbypass)
1270 //{
1271 // chopBypass(url, infectionbypass ? "GIBYPASS=" : "GBYPASS=");
1272
1273 //if (url.contains(infectionbypass ? "GIBYPASS=" : "GBYPASS=")) {
1274 //if (url.contains(infectionbypass ? "?GIBYPASS=" : "?GBYPASS=")) {
1275 //String bypass(url.after(infectionbypass ? "?GIBYPASS=" : "?GBYPASS="));
1276 ////header.front() = header.front().before(infectionbypass ? "?GIBYPASS=" : "?GBYPASS=") + header.front().after(bypass.toCharArray());
1277 //} else {
1278 //String bypass(url.after(infectionbypass ? "&GIBYPASS=" : "&GBYPASS="));
1279 //header.front() = header.front().before(infectionbypass ? "&GIBYPASS=" : "&GBYPASS=") + header.front().after(bypass.toCharArray());
1280 //}
1281 //}
1282 //cachedurl = "";
1283 //}
1284
1285 // same for scan bypass
1286 //void HTTPHeader::chopScanBypass(String url)
1287 //{
1288 // chopBypass(url,"GSBYPASS=");
1289 // if (url.contains("GSBYPASS=")) {
1290 // if (url.contains("?GSBYPASS=")) {
1291 // String bypass(url.after("?GSBYPASS="));
1292 // header.front() = header.front().before("?GSBYPASS=") + header.front().after(bypass.toCharArray());
1293 // } else {
1294 // String bypass(url.after("&GSBYPASS="));
1295 // header.front() = header.front().before("&GSBYPASS=") + header.front().after(bypass.toCharArray());
1296 // }
1297 // }
1298 // cachedurl = "";
1299 //}
1300
1301 // New generic chopBypass - allows new types to be added
1302 void HTTPHeader::chopBypass(String url,std::string btype) // Note: type must include = , e.g. GBYPASS=
1303 {
1304 if (url.contains(reinterpret_cast<const char *>(btype.c_str()))) {
1305 std::string qtype("?");
1306 qtype += btype; // = ?GIBYPASS= or ?GBYPASS= etc
1307 if (url.contains(reinterpret_cast<const char *>(qtype.c_str()))) {
1308 String bypass(url.after(qtype.c_str()));
1309 String after(header.front().after(bypass.toCharArray()));
1310 header.front() = header.front().before(qtype.c_str());
1311 header.front() += after;
1312 } else {
1313 std::string qtype("&");
1314 qtype += btype; // = &GIBYPASS= or &GBYPASS= etc
1315 if (url.contains(reinterpret_cast<const char *>(qtype.c_str()))) {
1316 String bypass(url.after(qtype.c_str()));
1317 String after(header.front().after(bypass.toCharArray()));
1318 header.front() = header.front().before(qtype.c_str());
1319 header.front() += after;
1320 }
1321 }
1322 cachedurl = "";
1323 }
1324 }
1325
1326
1327 // I'm not proud of this... --Ernest
1328 String HTTPHeader::getCookie(const char *cookie)
1329 {
1330 String line;
1331 // TODO - do away with loop here somehow, or otherwise speed it up?
1332 for (std::deque<String>::iterator i = header.begin(); i != header.end(); i++) {
1333 if (i->startsWithLower("cookie:")) {
1334 line = i->after(": ");
1335 if (line.contains(cookie)) { // We know we have the cookie
1336 line = line.after(cookie);
1337 line.lop(); // Get rid of the '='
1338 if (line.contains(";")) {
1339 line = line.before(";");
1340 }
1341 }
1342 // break; // Technically there should be only one Cookie: header, but...
1343 }
1344 }
1345 line.removeWhiteSpace();
1346 DEBUG_debug("Found cookie:", line );
1347 return line;
1348 }
1349
1350 // add cookie with given name & value to outgoing headers
1351 void HTTPHeader::setCookie(const char *cookie, const char *domain, const char *value)
1352 {
1353 String line("Set-Cookie: ");
1354 line += cookie;
1355 line += "=";
1356 line += value;
1357 line += "; path=/; domain=.";
1358 line += domain;
1359 line += "\r";
1360 header.push_back(line);
1361 DEBUG_debug("Setting cookie:", line);
1362 // no expiry specified so ends with the browser session
1363 }
1364
1365 // is this a temporary filter bypass cookie?
1366 bool HTTPHeader::isBypassCookie(String url, const char *magic, const char *clientip, const char *user)
1367 {
1368 String cookie(getCookie("GBYPASS"));
1369 if (!cookie.length()) {
1370 DEBUG_debug("No bypass cookie");
1371 return false;
1372 }
1373 String cookiehash(cookie.subString(0, 32));
1374 String cookietime(cookie.after(cookiehash.toCharArray()));
1375 String mymagic(magic);
1376 mymagic += clientip;
1377 mymagic += user;
1378 mymagic += cookietime;
1379 bool matched = false;
1380 while (url.contains(".")) {
1381 String hashed(url.md5(mymagic.toCharArray()));
1382 DEBUG_debug("Bypass cookie:", cookiehash, " hashed: ", hashed, " contains ", clientip, " ", user, " ", url, " ", cookietime);
1383 if (hashed == cookiehash) {
1384 matched = true;
1385 break;
1386 }
1387 url = url.after(".");
1388 }
1389 if (not matched) {
1390 DEBUG_debug("Cookie GBYPASS not match");
1391 return false;
1392 }
1393 time_t timen = time(NULL);
1394 time_t timeu = cookietime.toLong();
1395 if (timeu < timen) {
1396 DEBUG_debug("Cookie GBYPASS expired: ", timeu, " ", timen );
1397 return false;
1398 }
1399 return true;
1400 }
1401
1402 String HTTPHeader::getReferer()
1403 {
1404 String line;
1405 for (std::deque<String>::iterator i = header.begin(); i != header.end(); i++) {
1406 if (i->startsWithLower("referer:")) {
1407 line = i->after(": ");
1408 break;
1409 }
1410 }
1411 line.removeWhiteSpace();
1412 DEBUG_debug("Found Referer URL:", line);
1413 return line;
1414 }
1415
1416 // *
1417 // *
1418 // * URL and Base64 decoding funcs
1419 // *
1420 // *
1421
1422 // URL decoding (%xx)
1423 // uses regex pre-compiled on startup
1424 String HTTPHeader::decode(const String &s, bool decodeAll)
1425 {
1426 if (s.length() < 3) {
1427 return s;
1428 }
1429 DEBUG_trace("decoding url");
1430 RegResult Rre;
1431 if (!urldecode_re.match(s.c_str(),Rre)) {
1432 return s;
1433 } // exit if not found
1434
1435 int match;
1436 int offset;
1437 int pos = 0;
1438 int size = s.length();
1439 String result;
1440 String n;
1441 for (match = 0; match < Rre.numberOfMatches(); match++) {
1442 offset = Rre.offset(match);
1443 if (offset > pos) {
1444 result += s.subString(pos, offset - pos);
1445 }
1446 n = Rre.result(match).c_str();
1447 n.lop(); // remove %
1448 result += hexToChar(n, decodeAll);
1449 // DEBUG_debug("encoded: ", Rre.result(match), " decoded: ", hexToChar(n), " string so far: ", result);
1450 pos = offset + 3;
1451 }
1452 if (size > pos) {
1453 result += s.subString(pos, size - pos);
1454 } else {
1455 n = "%" + n;
1456 }
1457 return result;
1458 }
1459
1460 // turn %xx back into original character
1461 String HTTPHeader::hexToChar(const String &n, bool all)
1462 {
1463 if (n.length() < 2) {
1464 return String(n);
1465 }
1466 static char buf[2];
1467 unsigned int a, b;
1468 unsigned char c;
1469 a = n[0];
1470 b = n[1];
1471 if (a >= 'a' && a <= 'f') {
1472 a -= 87;
1473 } else if (a >= 'A' && a <= 'F') {
1474 a -= 55;
1475 } else if (a >= '0' && a <= '9') {
1476 a -= 48;
1477 } else {
1478 return String("%") + n;
1479 }
1480 if (b >= 'a' && b <= 'f') {
1481 b -= 87;
1482 } else if (b >= 'A' && b <= 'F') {
1483 b -= 55;
1484 } else if (b >= '0' && b <= '9') {
1485 b -= 48;
1486 } else {
1487 return String("%") + n;
1488 }
1489 c = a * 16 + b;
1490 if (all || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || (c == '-')) {
1491 buf[0] = c;
1492 buf[1] = '\0';
1493 return String(buf);
1494 } else {
1495 return String("%") + n;
1496 }
1497 }
1498
1499 // decode a line of base64
1500 std::string HTTPHeader::decodeb64(const String &line)
1501 { // decode a block of b64 MIME - will not work with binary data only with text - trailing cr or newline are ignored
1502 long four = 0;
1503 int d;
1504 std::string result;
1505 int len = line.length() - 3;
1506 for (int i = 0; i < len; i += 4) {
1507 four = 0;
1508 d = decode1b64(line[i + 0]);
1509 four = four | d;
1510 d = decode1b64(line[i + 1]);
1511 four = (four << 6) | d;
1512 d = decode1b64(line[i + 2]);
1513 four = (four << 6) | d;
1514 d = decode1b64(line[i + 3]);
1515 four = (four << 6) | d;
1516 d = (four & 0xFF0000) >> 16;
1517 result += (char)d; // first char cannot be padding so this is safe
1518 d = (four & 0xFF00) >> 8;
1519 if (d > 0) { // if d is 0 = padding so do not put ^@ in string!
1520 result += (char)d;
1521 }
1522 d = four & 0xFF;
1523 if (d > 0) result += (char)d; // if d is 0 = padding so do not put ^@ in string!
1524 }
1525 return result;
1526 }
1527
1528 // decode an individual base64 character
1529 int HTTPHeader::decode1b64(char c)
1530 {
1531 unsigned char i = '\0';
1532 switch (c) {
1533 case '+':
1534 i = 62;
1535 break;
1536 case '/':
1537 i = 63;
1538 break;
1539 case '=':
1540 i = 0;
1541 break;
1542 default: // must be A-Z, a-z or 0-9
1543 i = '9' - c;
1544 if (i > 0x3F) { // under 9
1545 i = 'Z' - c;
1546 if (i > 0x3F) { // over Z
1547 i = 'z' - c;
1548 if (i > 0x3F) { // over z so invalid
1549 i = 0x80; // so set the high bit
1550 } else {
1551 // a-z
1552 i = c - 71;
1553 }
1554 } else {
1555 // A-Z
1556 i = c - 65;
1557 }
1558 } else {
1559 // 0-9
1560 i = c + 4;
1561 }
1562 break;
1563 }
1564 return (int)i;
1565 }
1566
1567 // *
1568 // *
1569 // * encode funcs
1570 // *
1571
1572 // get encoded URL?
1573 String HTTPHeader::URLEncode()
1574 {
1575 std::string encoded;
1576 String newurl(getUrl());
1577 const char *s = newurl.c_str();
1578 char *buf = new char[3];
1579 unsigned char c;
1580 for (int i = 0; i < (signed)strlen(s); i++) {
1581 c = s[i];
1582 // allowed characters in a url that have non special meaning
1583 if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) {
1584 encoded += c;
1585 continue;
1586 }
1587 // all other characters get encoded
1588 sprintf(buf, "%02x", c);
1589 encoded += "%";
1590 encoded += buf;
1591 }
1592 delete[] buf;
1593 String returnS(encoded);
1594 return returnS;
1595 }
1596
1597 String HTTPHeader::stringHeader() {
1598 String l;
1599 DEBUG_debug("stringHeader started hsize=", header.size() );
1600 if (header.size() > 0) {
1601 for (std::deque<String>::iterator i = header.begin(); i != header.end(); i++) {
1602 if (! (*i).startsWith("X-E2G-IgnoreMe")){
1603 DEBUG_debug("Found Header: ", *i );
1604 l += (*i) + "\n";
1605 }
1606 else {
1607 DEBUG_debug("Found Header X-E2G-IgnoreMe: ", *i );
1608 }
1609 }
1610 l += "\r\n";
1611 }
1612 return l;
1613 }
1614
1615 // *
1616 // *
1617 // * network send/receive funcs
1618 // *
1619 // *
1620
1621 // send headers out over the given socket
1622 // "reconnect" flag gives permission to reconnect to the socket on write error
1623 // - this allows us to re-open the proxy connection on pconns if squid's end has
1624 // timed out but the client's end hasn't. not much use with NTLM, since squid
1625 // will throw a 407 and restart negotiation, but works well with basic & others.
1626 //void HTTPHeader::out(Socket *peersock, Socket *sock, int sendflag, bool reconnect) throw(std::exception)
1627 bool HTTPHeader::out(Socket *peersock, Socket *sock, int sendflag, bool reconnect )
1628 {
1629 String l; // for amalgamating to avoid conflict with the Nagel algorithm
1630
1631 if (header_all_sent) return true;
1632
1633 if ((sendflag == __E2HEADER_SENDALL || sendflag == __E2HEADER_SENDFIRSTLINE) && !first_line_sent) {
1634 if (header.size() > 0) {
1635 l = header.front() + "\n";
1636 first_line_sent = true;
1637
1638 if(is_response) {
1639 DEBUG_debug("response headerout:", l);
1640 } else {
1641 DEBUG_debug("request headerout:", l);
1642 }
1643
1644 //if a socket is ssl we want to send relative paths not absolute urls
1645 //also HTTP responses dont want to be processed (if we are writing to an ssl client socket then we are doing a request)
1646 if (sock->isSsl() && !sock->isSslServer()) {
1647 setDirect();
1648 }
1649
1650 if (isdirect && !is_response) {
1651 l = header.front().before(" ") + " /" + header.front().after("://").after("/").before(" ") + " HTTP/1.1\r\n";
1652 DEBUG_debug("request headerout (modified for direct):", l);
1653 }
1654
1655 // first reconnect loop - send first line
1656 while (true) {
1657 if (!sock->writeToSocket(l.toCharArray(), l.length(), 0, timeout)) {
1658 // reconnect & try again if we've been told to
1659 if (reconnect && !isdirect) {
1660 // don't try more than once
1661 E2LOGGER_error( "Proxy connection broken (1); trying to re-establish...");
1662 reconnect = false;
1663 sock->reset();
1664 int rc = sock->connect(o.net.proxy_ip, o.net.proxy_port);
1665 if (rc)
1666 return false;
1667 // throw std::exception();
1668 continue;
1669 }
1670 // throw std::exception();
1671 return false;
1672 }
1673 // if we got here, we succeeded, so break the reconnect loop
1674 DEBUG_debug("headertoclient: ", l.substr(0, l.length()-1), " timeout:", String(timeout));
1675 break;
1676 }
1677 }
1678 if (sendflag == __E2HEADER_SENDFIRSTLINE) {
1679 return true;
1680 }
1681 }
1682
1683 l = "";
1684
1685 if (header.size() > 1) {
1686 for (std::deque<String>::iterator i = header.begin() + 1; i != header.end(); i++) {
1687 if (! (*i).startsWith("X-E2G-IgnoreMe")){
1688 DEBUG_debug("Found Header: ", *i);
1689 l += (*i) + "\n";
1690 }
1691 else {
1692 DEBUG_debug("Found Header X-E2G-IgnoreMe: ", *i );
1693 }
1694 }
1695
1696 }
1697 if (!is_response && o.header.forwarded_for && !isdirect) {
1698 std::string line("X-Forwarded-For: ");
1699 line.append(s_clientip).append("\r\n");
1700 DEBUG_debug("Adding Header: ", line);
1701 l += line;
1702 }
1703 l += "\r\n";
1704
1705 // second reconnect loop
1706 while (true) {
1707 // send header to the output stream
1708 // need exception for bad write
1709
1710 if (!sock->writeToSocket(l.toCharArray(), l.length(), 0, timeout)) {
1711 // reconnect & try again if we've been told to
1712 if (reconnect && !isdirect) {
1713 // don't try more than once
1714 E2LOGGER_error("Proxy connection broken (2); trying to re-establish...");
1715 reconnect = false;
1716 sock->reset();
1717 int rc = sock->connect(o.net.proxy_ip, o.net.proxy_port);
1718 if (rc)
1719 return false;
1720 // throw std::exception();
1721 // include the first line on the retry
1722 l = header.front() + "\n" + l;
1723 continue;
1724 }
1725 //throw std::exception();
1726 return false;
1727 }
1728 // if we got here, we succeeded, so break the reconnect loop
1729 break;
1730 }
1731 DEBUG_debug("Header written - pstdata_len:", postdata_len );
1732
1733 if (postdata_len > 0) {
1734 DEBUG_debug("Sending manually set POST data");
1735 if (!sock->writeToSocket(postdata, postdata_len, 0, timeout)) {
1736 DEBUG_debug("Could not send POST data!");
1737 //throw std::exception();
1738 return false;
1739 }
1740 } else if (!is_response && (peersock != nullptr) && ((contentlength > 0) || chunked) && !expects_100) {
1741 DEBUG_debug("Opening tunnel for POST data");
1742 FDTunnel fdt;
1743 if (!fdt.tunnel(*peersock, *sock, false, contentLength(), true, chunked) )
1744 return false;
1745 }
1746 DEBUG_debug("Returning from header:out ");
1747 #ifdef DEBUG_LOW
1748 dbshowheader(true);
1749 #endif
1750 header_all_sent = true;
1751 return true;
1752 }
1753
1754 // discard remainder of POST data
1755 void HTTPHeader::discard(Socket *sock, off_t cl)
1756 {
1757 static char header[4096];
1758 if (cl == -2)
1759 cl = contentLength();
1760 int rc;
1761 while (cl > 0) {
1762 rc = sock->readFromSocket(header, ((cl > 4096) ? 4096 : cl), 0, timeout);
1763 if (rc > 0)
1764 cl -= rc;
1765 else
1766 break;
1767 }
1768 }
1769
1770 void HTTPHeader::setClientIP(String &ip) {
1771 s_clientip = ip.toCharArray();
1772 }
1773
1774 String HTTPHeader::getClientIP() {
1775 return s_clientip;
1776 }
1777
1778 void HTTPHeader::setDirect() {
1779 isdirect = true;
1780 }
1781
1782 bool HTTPHeader::in_handle_100(Socket *sock, bool allowpersistent, bool expect_100) {
1783 int max_100s = 10;
1784 while( max_100s > 0)
1785 {
1786 if( in(sock,allowpersistent)) {
1787 if (returncode == 100) {
1788 if (expect_100) {
1789 return true;
1790 }
1791 // discard 100 continue header and get next header
1792 max_100s--;
1793 continue;
1794 } else { // all other return codes
1795 return true;
1796 }
1797 } else {
1798 return false;
1799 }
1800 }
1801 DEBUG_debug("Too many repeat 100 continues");
1802 return false;
1803 }
1804
1805 bool HTTPHeader::in(Socket *sock, bool allowpersistent)
1806 {
1807 if (dirty)
1808 reset();
1809 dirty = true;
1810
1811 #ifdef DEBUG_LOW
1812 if(is_response) {
1813 DEBUG_debug("Start of response header:in");
1814 }
1815 else {
1816 DEBUG_debug("Start of request header:in");
1817 }
1818 #endif
1819
1820 // the RFCs don't specify a max header line length so this should be
1821 // dynamic really. Pointed out (well reminded actually) by Daniel Robbins
1822 char buff[32768]; // setup a buffer to hold the incomming HTTP line
1823 String line; // temp store to hold the line after processing
1824 line = "----"; // so we get past the first while
1825 bool firsttime = true;
1826 bool discard = false;
1827 while (line.length() > 3 || discard) { // loop until the stream is
1828 // failed or we get to the end of the header (a line by itself)
1829
1830 // get a line of header from the stream
1831 // on the first time round the loop, honour the reloadconfig flag if desired
1832 // - this lets us break when waiting for the next request on a pconn, but not
1833 // during receipt of a request in progress.
1834 bool truncated = false;
1835 int rc;
1836 if (firsttime) {
1837 DEBUG_debug("header:in firsttime before getLine - timeout:", timeout );
1838 rc = sock->getLine(buff, 32768, timeout, NULL, &truncated);
1839 DEBUG_debug("firstime: header:in after getLine ");
1840 if (rc < 0 || truncated) {
1841 ispersistent = false;
1842 DEBUG_debug("firstime: header:in after getLine: rc: ", rc, " truncated: ", truncated );
1843 #ifdef DEBUG_LOW
1844 dbshowheader(false);
1845 #endif
1846 return false;
1847 }
1848 line = buff;
1849 header.push_back(line); // stick the line in the deque that holds the header
1850 if(!checkfirstheaderline()) {
1851 DEBUG_debug("firstime: checkfirstheaerline returned false ", header.front(), "pad " );
1852 header.clear();
1853 return false;
1854 }
1855 // Take this out as it appears that 100 continue can have other header lines and is terminated with a blank line like other status returns
1856 // if(is_response && returncode == 100) {
1857 //DEBUG_debug("firstime: return code is 100" );
1858 // return true;
1859 //}
1860 } else {
1861 rc = sock->getLine(buff, 32768, timeout, NULL, &truncated);
1862 if (rc < 0 || truncated) {
1863 ispersistent = false;
1864 DEBUG_debug("not firstime header:in after getLine: rc: ", rc, " truncated: ", truncated );
1865 #ifdef DEBUG_LOW
1866 dbshowheader(false);
1867 #endif
1868 return false; // do not allow non-terminated headers
1869 }
1870
1871 }
1872
1873 if (header.size() > o.header.max_header_lines) {
1874 E2LOGGER_error("header:size too big: %lu, see maxheaderlines", header.size());
1875 dbshowheader(false);
1876 ispersistent = false;
1877 return false;
1878 }
1879
1880 // throw std::exception();
1881
1882 // getline will throw an exception if there is an error which will
1883 // only be caught by HandleConnection() ?????????????????????
1884
1885 if (rc > 0 ) line = buff;
1886 else line = "";// convert the line to a String
1887
1888 if(firsttime && is_response) {
1889 // check first line header
1890 if (!(line.length() > 11 && line.startsWith("HTTP/") && (line.after(" ").before(" ").toInteger() > 99)))
1891 {
1892 if(o.conn.logconerror)
1893 E2LOGGER_error("Returning from header:in Server did not respond with HTTP ");
1894 #ifdef DEBUG_LOW
1895 dbshowheader(false);
1896 #endif
1897 return false;
1898 }
1899 }
1900 // ignore crap left in buffer from old pconns (in particular, the IE "extra CRLF after POST" bug)
1901 discard = false;
1902 //if ((firsttime && line.length() <= 3)) {
1903 // discard = true;
1904 // DEBUG_debug("Discarding unwanted bytes at head of request (pconn closed or IE multipart POST bug)");
1905 //} else {
1906 // header.push_back(line); // stick the line in the deque that holds the header
1907 // }
1908 if(firsttime) {
1909 firsttime = false;
1910 } else {
1911 header.push_back(line); // stick the line in the deque that holds the header
1912 }
1913 // End of while
1914 }
1915
1916 // At this point there should be at least two header lines including the final blank line - should fix #
1917 if (header.size() < 2) {
1918 DEBUG_debug("header:size < 2 - incomplete ");
1919 return false;
1920 }
1921
1922 header.pop_back(); // remove the final blank line of a header
1923
1924 DEBUG_debug("header:size = ", header.size());
1925 if (header.size() > 0) {
1926 DEBUG_debug("first line = ", header[0]);
1927 }
1928
1929 checkheader(allowpersistent); // sort out a few bits in the header
1930
1931 DEBUG_debug("isProxyRequest is ", isProxyRequest);
1932 return true;
1933 }