Debian Codegraph @

src/DataBuffer.cpp

raw
1 // For all support, instructions and copyright go to:
2 // http://e2guardian.org/
3 // Released under the GPL v2, with the OpenSSL exception described in the README file.
4
5 // INCLUDES
6
7 #ifdef HAVE_CONFIG_H
8 #include "e2config.h"
9 #endif
10 #include "DataBuffer.hpp"
11 #include "HTTPHeader.hpp"
12 #include "OptionContainer.hpp"
13 #include "Logger.hpp"
14
15 #include <sys/stat.h>
16 #include <algorithm>
17 #include <cstdlib>
18 #include <unistd.h>
19 #include <zlib.h>
20 #include <cerrno>
21 #include <fstream>
22 #include <sys/time.h>
23 #include <queue>
24 #include <istream>
25
26 // DEFINES
27
28 #define __E2HEADER_SENDALL 0
29 #define __E2HEADER_SENDFIRSTLINE 1
30 #define __E2HEADER_SENDREST 2
31
32 // GLOBALS
33
34 extern OptionContainer o;
35
36 // IMPLEMENTATION
37
38 DataBuffer::DataBuffer()
39 : data(new char[1]), data_length(0)
40 {
41 data[0] = '\0';
42 }
43
44 DataBuffer::DataBuffer(const void *indata, off_t length) //not used! PIP
45 : data(new char[length]), data_length(length)
46 {
47 memcpy(data, indata, length);
48 }
49
50 void DataBuffer::reset()
51 {
52 delete[] data;
53 delete[] compresseddata;
54
55 data = new char[1];
56 data[0] = '\0';
57
58
59 compresseddata = nullptr;
60 data_length = 0;
61 buffer_length = 0;
62 compressed_buffer_data_length = 0;
63
64 if (tempfilefd > -1) {
65 close(tempfilefd);
66 if (!preservetemp) {
67 unlink(tempfilepath.toCharArray());
68 }
69 tempfilefd = -1;
70 tempfilesize = 0;
71 }
72
73 bytesalreadysent = 0;
74 dontsendbody = false;
75 preservetemp = false;
76 decompress = "";
77 chunked = false;
78 icap = false;
79 }
80
81 // delete the memory block when the class is destroyed
82 DataBuffer::~DataBuffer()
83 {
84 delete[] data;
85 if (compresseddata != nullptr) {
86 delete[] compresseddata;
87 compresseddata = nullptr;
88 }
89 if (tempfilefd > -1) {
90 close(tempfilefd);
91 if (!preservetemp) {
92 unlink(tempfilepath.toCharArray());
93 }
94 tempfilefd = -1;
95 tempfilesize = 0;
96 }
97 }
98
99 void DataBuffer::set_current_config (FOptionContainer *newfgc) {
100 fgc = newfgc;
101 }
102
103 //}
104 // swap back to a compressed version of the data, if one exists
105 // also delete uncompressed version
106 // if body was decompressed but not modified, this can save bandwidth
107 void DataBuffer::swapbacktocompressed()
108 {
109 if (compresseddata != nullptr && compressed_buffer_data_length> 0) {
110 delete[] data;
111 data_length = compressed_buffer_data_length;
112 data = compresseddata;
113
114 compresseddata = nullptr;
115 compressed_buffer_data_length = 0;
116 DEBUG_network("Compressed data size ", data_length);
117 }
118 }
119
120 int DataBuffer::readInFromSocket(Socket *sock, int size, bool wantall, int &result) {
121
122 int rc;
123
124 DEBUG_trace("size:", size, " wantall:", wantall);
125 if (size < 1) {
126 E2LOGGER_error("read request is negative");
127 return -1;
128 }
129
130 if (wantall) {
131 if (!swappedtodisk) {
132 // if not swapped to disk and file is too large for RAM, then swap to disk
133 if (data_length > o.content.max_content_ramcache_scan_size) {
134 DEBUG_debug("swapping to disk");
135 tempfilefd = getTempFileFD();
136 if (tempfilefd < 0) {
137 E2LOGGER_error("error buffering to disk so skipping disk buffering");
138 result = DB_TOBIG;
139 return -1;
140 }
141 write(tempfilefd, data, data_length);
142 swappedtodisk = true;
143 tempfilesize = data_length;
144 }
145 } else if (tempfilesize > o.content.max_content_filecache_scan_size) {
146 // if swapped to disk and file too large for that too, then give up
147 DEBUG_debug("defaultdm: file too big to be scanned, halting download");
148 result = DB_TOBIG | DB_TOBIG_SCAN;
149 return -1;
150 }
151 } else {
152 if (data_length > o.content.max_content_filter_size) {
153 // if we aren't downloading for virus scanning, and file too large for filtering, give up
154 DEBUG_debug("defaultdm: file too big to be filtered, halting download");
155 result = DB_TOBIG | DB_TOBIG_FILTER;
156 return -1;
157 }
158 }
159
160 DEBUG_debug("swappedtodisk:", swappedtodisk);
161 if (!swappedtodisk) {
162 if (size > (buffer_length - data_length)) {
163 if(!increase_buffer(size - (buffer_length - data_length))) {
164 size = (buffer_length - data_length);
165 if (size == 0) {
166 result = DB_TOBIG;
167 return -1; // it is too big
168 }
169 }
170 }
171 if (chunked) {
172 DEBUG_debug("readChunk:", data_length, ",", size);
173 rc = sock->readChunk((data + data_length), size, timeout);
174 } else {
175 DEBUG_debug("bufferReadFromSocket:", data_length, ",", size);
176 rc = bufferReadFromSocket(sock, (data + data_length), size, timeout);
177 }
178
179 if (rc <= 0) {
180 if (chunked)
181 got_all = true;
182 return -1;
183 // an error occurred so end the while()
184 // or none received so pipe iis closed or chunking has ended
185 } else {
186 bytes_toget -= rc;
187 data_length += rc;
188 data[data_length] = '\0';
189 }
190 } else {
191 if (chunked) {
192 rc = sock->readChunk(data, buffer_length, timeout);
193 } else {
194 rc = bufferReadFromSocket(sock, data,
195 // if not getting everything until connection close, grab only what is left
196 (!geteverything && (bytes_toget < buffer_length) ? bytes_toget
197 : buffer_length),
198 timeout);
199 }
200 if (rc <= 0) {
201 if (chunked)
202 got_all = true;
203 result = 0;
204 return 0;
205 } else {
206 bytes_toget -= rc;
207 write(tempfilefd, data, rc);
208 tempfilesize += rc;
209 DEBUG_debug("written to disk:", rc, " total:", tempfilesize);
210 }
211 }
212 result = 0;
213 DEBUG_debug("rc=", rc, " bytes_toget=", bytes_toget, " data_length=", data_length);
214 return rc;
215 }
216
217 bool DataBuffer::increase_buffer(int extra) {
218 int more = 65536;
219 if (extra > more)
220 more = extra;
221 if ((buffer_length + more) > o.content.max_content_filter_size) {
222 more = o.content.max_content_filter_size - buffer_length;
223 }
224 if (more > 0) {
225 char *temp = new char[buffer_length + more + 1]; // replacement store
226 temp[buffer_length + more] = '\0';
227 memcpy(temp, data, data_length); // copy the current data
228 delete[] data; // delete the current data block
229 data = temp;
230 temp = nullptr;
231 buffer_length += more; // update data size counter
232 DEBUG_debug("data buffer extended by ", more, " to ", buffer_length);
233 return true;
234 }
235 return false;
236 }
237
238 // a much more efficient reader that does not assume the contents of
239 // the buffer gets filled thus reducing memcpy()ing and new()ing
240 int DataBuffer::bufferReadFromSocket(Socket *sock, char *buffer, int size, int sockettimeout)
241 {
242 int pos = 0;
243 int rc;
244 while (pos < size) {
245 rc = sock->readFromSocket(&buffer[pos], size - pos, 0, sockettimeout, true);
246 if (rc < 1) {
247 // none recieved or an error
248 if (pos > 0) {
249 return pos; // some was recieved previous into buffer
250 }
251 return rc; // just return with the return code
252 }
253 pos += rc;
254 }
255 return size; // full buffer
256 }
257
258 // a much more efficient reader that does not assume the contents of
259 // the buffer gets filled thus reducing memcpy()ing and new()ing.
260 // in addition to the actual socket timeout, used for each individual read, this version
261 // incorporates a "global" timeout within which all reads must complete.
262 int DataBuffer::bufferReadFromSocket(Socket *sock, char *buffer, int size, int sockettimeout, int stimeout)
263 {
264 DEBUG_trace("");
265 int pos = 0;
266 int rc;
267 struct timeval starttime;
268 struct timeval nowadays;
269 gettimeofday(&starttime, NULL);
270 while (pos < size) {
271 if (chunked) {
272 rc = sock->readChunk(&buffer[pos], size - pos,sockettimeout );
273 } else {
274 rc = sock->readFromSocket(&buffer[pos], size - pos, 0, sockettimeout, true);
275 }
276 if (rc < 1) {
277 // none recieved or an error
278 if (pos > 0) {
279 return pos; // some was recieved previous into buffer
280 }
281 return rc; // just return with the return code
282 }
283 pos += rc;
284 gettimeofday(&nowadays, NULL);
285 if (nowadays.tv_sec - starttime.tv_sec > stimeout) {
286 DEBUG_debug("buffered socket read more than timeout");
287 return pos; // just return how much got so far then
288 }
289 }
290 return size; // full buffer
291 }
292
293 // make a temp file and return its FD. only currently used in DM plugins.
294 int DataBuffer::getTempFileFD()
295 {
296 if (tempfilefd > -1) {
297 return tempfilefd;
298 }
299 tempfilepath = o.content.download_dir.c_str();
300 tempfilepath += "/tfXXXXXX";
301 char *tempfilepatharray = new char[tempfilepath.length() + 1];
302 strcpy(tempfilepatharray, tempfilepath.toCharArray());
303 umask(0007); // only allow access to e2g user and group
304 if ((tempfilefd = mkstemp(tempfilepatharray)) < 0) {
305 E2LOGGER_error("Could not create temp file to store download for scanning: ", strerror(errno));
306 tempfilefd = -1;
307 tempfilepath = "";
308 } else {
309 tempfilepath = tempfilepatharray;
310 }
311 delete[] tempfilepatharray;
312 return tempfilefd;
313 }
314
315 // check the client's user agent, see if we have a DM plugin compatible with it, and use it to download the body of the given request
316 bool DataBuffer::in(Socket *sock, Socket *peersock, HTTPHeader *requestheader, HTTPHeader *docheader, bool runav, int *headersent,StoryBoard &story, NaughtyFilter *cm)
317 {
318 //Socket *sock = where to read from
319 //Socket *peersock = browser to send stuff to for keeping it alive
320 //HTTPHeader *requestheader = header client used to request
321 //HTTPHeader *docheader = header used for sending first line of reply
322 //bool runav = to determine if limit is av or not
323 //int *headersent = to use to send the first line of header if needed
324 // or to mark that the header has already been sent
325
326 // so we know if we only partially downloaded from
327 // upstream so later, if allowed, we can send the rest
328 bool toobig = false;
329
330 // match request to download manager so browsers potentially can have a prettier version
331 // and software updates, stream clients, etc. can have a compatible version.
332 //int rc = 0;
333 int j = 0;
334 // int rc = -1;
335 DEBUG_trace("");
336 for (std::deque<Plugin *>::iterator i = o.plugins.dmplugins_begin; i != o.plugins.dmplugins_end; i++) {
337 ++j;
338 if ((i + 1) == o.plugins.dmplugins_end) {
339 DEBUG_debug("Got to final download manager so defaulting to always match.");
340 dm_plugin = (DMPlugin *)(*i);
341 dm_plugin->in(this, sock, peersock, requestheader, docheader, runav, headersent, &toobig);
342 break;
343 } else {
344 dm_plugin = (DMPlugin *)(*i);
345 if (story.runFunctEntry(dm_plugin->story_entry, *cm)) {
346 DEBUG_debug("Matching download manager number: ", j);
347 dm_plugin->in(this, sock, peersock, requestheader, docheader, runav, headersent, &toobig);
348 break;
349 }
350 }
351 j++;
352 }
353 return toobig;
354 }
355
356 // send the request body to the client after having been handled by a DM plugin
357 bool DataBuffer::out(Socket *sock)
358 {
359 if (dontsendbody) {
360 DEBUG_debug("dontsendbody true; not sending");
361 return true;
362 }
363 // if (!(*sock).readyForOutput(timeout)) return false; // exceptions on timeout or error
364
365 DEBUG_trace("");
366 if (tempfilefd > -1) {
367 // must have been too big for ram so stream from disk in blocks
368 DEBUG_debug("Sending ", tempfilesize - bytesalreadysent, " bytes from temp file (", bytesalreadysent, " already sent)");;
369 off_t sent = bytesalreadysent;
370 int rc;
371
372 if (lseek(tempfilefd, bytesalreadysent, SEEK_SET) < 0)
373 return false;
374 int block_len;
375 if(chunked)
376 block_len = 4048;
377 else
378 block_len = data_length;
379
380 while (sent < tempfilesize) {
381 rc = read(tempfilefd, data, block_len);
382 DEBUG_debug("reading temp file rc:", rc);
383 if (rc < 0) {
384 DEBUG_debug("error reading temp file so throwing exception");
385 return false;
386 }
387 if (rc == 0) {
388 DEBUG_debug("got zero bytes reading temp file");
389 break; // should never happen
390 }
391 // as it's cached to disk the buffer must be reasonably big
392 if(chunked) {
393 if (!sock->writeChunk(data, rc, timeout))
394 return false;
395 } else {
396 if (!sock->writeToSocket(data, rc, 0, timeout)) {
397 return false;
398 }
399 }
400 sent += rc;
401 DEBUG_debug("total sent from temp:", sent);
402 }
403 if (chunked && got_all) {
404 String n;
405 if (!sock->writeChunkTrailer(n))
406 return false;
407 }
408 close(tempfilefd);
409 tempfilefd = -1;
410 tempfilesize = 0;
411 unlink(tempfilepath.toCharArray());
412 } else {
413 off_t sent = bytesalreadysent;
414 DEBUG_debug("Sending ", data_length, " bytes from RAM (", buffer_length, " in buffer; ", bytesalreadysent, " already sent)" );
415 // it's in RAM, so just send it, no streaming from disk
416 int block_len;
417 if(chunked)
418 block_len = 4048;
419 else
420 block_len = data_length;
421
422 if (data_length != 0) {
423 while (sent < data_length) {
424 if( block_len > (data_length - sent))
425 block_len = (data_length - sent);
426 if (chunked) {
427 if (!sock->writeChunk(data + sent, block_len, timeout)) {
428 DEBUG_network("writeChunk failed after ", sent, " bytes");
429 return false;
430 }
431
432 } else {
433 if (!sock->writeToSocket(data + sent, data_length - sent, 0, timeout))
434 return false;
435 }
436 sent += block_len;
437 }
438 if (chunked && got_all) {
439 String n;
440 if (!sock->writeChunkTrailer(n))
441 return false;
442 }
443 } else {
444 if(chunked) {
445 if (!sock->writeChunk(data + bytesalreadysent, 0, timeout))
446 return false;
447 } else {
448 if (!sock->writeToSocket("\r\n\r\n", 4, 0, timeout))
449 return false;
450 }
451 }
452 DEBUG_debug("Sent ", data_length - bytesalreadysent," bytes from RAM (", buffer_length);
453 }
454 return true;
455 }
456
457 // zlib decompression
458 void DataBuffer::zlibinflate(bool header)
459 {
460 if (data_length < 12) {
461 return; // it can't possibly be zlib'd
462 }
463 DEBUG_debug("compressed size:", data_length);
464
465 #if ZLIB_VERNUM < 0x1210
466 #warning ************************************
467 #warning For gzip support you need zlib 1.2.1
468 #warning or later to be installed.
469 #warning You can ignore this warning but
470 #warning internet bandwidth may be wasted.
471 #warning ************************************
472 if (header) {
473 return;
474 }
475 #endif
476
477 int newsize = data_length * 5; // good estimate of deflated HTML
478
479 char *block = new char[newsize];
480 block[0] = '\0';
481
482 char *temp = NULL;
483
484 off_t bytesgot = 0;
485 int err;
486
487 z_stream d_stream;
488 d_stream.zalloc = (alloc_func)0;
489 d_stream.zfree = (free_func)0;
490 d_stream.opaque = (voidpf)0;
491 d_stream.next_in = (Bytef *)data;
492 d_stream.avail_in = data_length;
493 d_stream.next_out = (Bytef *)block;
494 d_stream.avail_out = newsize;
495
496 // inflate either raw zlib, or possibly gzip with a header
497 if (header) {
498 err = inflateInit2(&d_stream, 15 + 32);
499 } else {
500 err = inflateInit2(&d_stream, -15);
501 }
502
503 if (err != Z_OK) { // was a problem so just return
504 delete[] block; // don't forget to free claimed memory
505 DEBUG_debug("bad init inflate: ", err);
506 return;
507 }
508 while (true) {
509 DEBUG_debug("inflate loop");
510 err = inflate(&d_stream, Z_SYNC_FLUSH);
511 bytesgot = d_stream.total_out;
512 if (err == Z_STREAM_END) {
513 err = inflateEnd(&d_stream);
514 if (err != Z_OK) {
515 delete[] block;
516 DEBUG_debug("bad inflateEnd: ", d_stream.msg);
517 return;
518 }
519 break;
520 }
521 if (err != Z_OK) { // was a problem so just return
522 delete[] block; // don't forget to free claimed memory
523 DEBUG_debug("bad inflate: ", d_stream.msg);
524 err = inflateEnd(&d_stream);
525 if (err != Z_OK) {
526 DEBUG_debug("bad inflateEnd: ", d_stream.msg);
527 }
528 return;
529 }
530 if (bytesgot > o.content.max_content_filter_size) {
531 delete[] block; // don't forget to free claimed memory
532 DEBUG_debug("inflated file larger than maxcontentfiltersize, not inflating further");
533 err = inflateEnd(&d_stream);
534 if (err != Z_OK) {
535 DEBUG_debug("bad inflateEnd: ", d_stream.msg);
536 }
537 return;
538 }
539
540 // inflation is going ok, but we don't have enough room in the output buffer
541 newsize = bytesgot * 2;
542 temp = new char[newsize];
543 memcpy(temp, block, bytesgot);
544 delete[] block;
545 block = temp;
546 temp = NULL;
547
548 d_stream.next_out = (Bytef *)(block + bytesgot);
549 d_stream.avail_out = newsize - bytesgot;
550 }
551
552 compresseddata = data;
553 compressed_buffer_data_length = data_length; // change from buffer_length
554 buffer_length = bytesgot;
555 DEBUG_debug("decompressed size: ", data_length);
556 data = new char[bytesgot + 1];
557 data[bytesgot] = '\0';
558 memcpy(data, block, bytesgot);
559 data_length = bytesgot;
560 delete[] block;
561 }
562
563 // Does a regexp search and replace.
564 struct newreplacement {
565 int match;
566 String replacement;
567 };
568 bool DataBuffer::contentRegExp(FOptionContainer* &foc)
569 {
570
571 DEBUG_debug("Starting content reg exp replace");
572 bool contentmodified = false;
573 unsigned int i;
574 unsigned int j, k, m;
575 unsigned int s = (*foc).content_regexp_list_comp.size();
576 unsigned int matches;
577 unsigned int submatch, submatches;
578 RegExp *re;
579 RegResult Rre;
580 String *replacement;
581 unsigned int replen;
582 int sizediff;
583 char *newblock;
584 char *dstpos;
585 unsigned int srcoff;
586 unsigned int nextoffset;
587 unsigned int matchlen;
588
589 std::queue<newreplacement *> matchqueue;
590
591 for (i = 0; i < s; i++) {
592 re = &((*foc).content_regexp_list_comp[i]);
593 if (re->match(data, Rre)) {
594 replacement = &((*foc).content_regexp_list_rep[i]);
595 //replen = replacement->length();
596 matches = Rre.numberOfMatches();
597
598 sizediff = 0;
599 m = 0;
600 for (j = 0; j < matches; j++) {
601 srcoff = Rre.offset(j);
602 matchlen = Rre.length(j);
603
604 // Count matches for ()'s
605 for (submatches = 0; j + submatches + 1 < matches; submatches++)
606 if (Rre.offset(j + submatches + 1) + Rre.length(j + submatches + 1) > srcoff + matchlen)
607 break;
608
609 // \1 and $1 replacement
610
611 // store match no. and default (empty) replacement string
612 newreplacement *newrep = new newreplacement;
613 newrep->match = j;
614 newrep->replacement = "";
615 // iterate over regex's replacement string
616 for (k = 0; k < replacement->length(); k++) {
617 // look for \1..\9 and $1..$9
618 if (((*replacement)[k] == '\\' || (*replacement)[k] == '$') && (*replacement)[k + 1] >= '1' && (*replacement)[k + 1] <= '9') {
619 // determine match number
620 submatch = (*replacement)[++k] - '0';
621 // add submatch contents to replacement string
622 if (submatch <= submatches) {
623 newrep->replacement += Rre.result(j + submatch).c_str();
624 }
625 } else {
626 // unescape \\ and \$, and add other non-backreference characters
627 if ((*replacement)[k] == '\\' && ((*replacement)[k + 1] == '\\' || (*replacement)[k + 1] == '$'))
628 k++;
629 newrep->replacement += replacement->subString(k, 1);
630 }
631 }
632 matchqueue.push(newrep);
633
634 // update size difference between original and modified content
635 sizediff -= Rre.length(j);
636 sizediff += newrep->replacement.length();
637 // skip submatches to next top level match
638 j += submatches;
639 m++;
640 }
641
642 // now we know eventual size of content-replaced block, allocate memory for it
643 newblock = new char[data_length + sizediff + 1];
644 newblock[data_length + sizediff] = '\0';
645 srcoff = 0;
646 dstpos = newblock;
647 matches = m;
648
649 DEBUG_debug("content matches:", matches);
650 // replace top-level matches using filled-out replacement strings
651 newreplacement *newrep;
652 for (j = 0; j < matches; j++) {
653 newrep = matchqueue.front();
654 nextoffset = Rre.offset(newrep->match);
655 if (nextoffset > srcoff) {
656 memcpy(dstpos, data + srcoff, nextoffset - srcoff);
657 dstpos += nextoffset - srcoff;
658 srcoff = nextoffset;
659 }
660 replen = newrep->replacement.length();
661 memcpy(dstpos, newrep->replacement.toCharArray(), replen);
662 dstpos += replen;
663 srcoff += Rre.length(newrep->match);
664 delete newrep;
665 matchqueue.pop();
666 }
667 if (srcoff < data_length) {
668 memcpy(dstpos, data + srcoff, data_length - srcoff);
669 }
670 delete[] data;
671 data = newblock;
672 data_length = data_length + sizediff;
673 contentmodified = true;
674 }
675 }
676 return contentmodified;
677 }
678
679 void DataBuffer::setChunked(bool ch = true) {
680 chunked = ch;
681 return;
682 }
683
684 void DataBuffer::setICAP(bool ch = true) {
685 chunked = ch;
686 icap = ch;
687 return;
688 }