Debian Codegraph @

src/NaughtyFilter.cpp

raw
1 // For all support, instructions and copyright go to:
2 // http://e2guardian.org/
3 // Released under the GPL v2, with the OpenSSL exception described in the README file.
4
5 // INCLUDES
6
7 #ifdef HAVE_CONFIG_H
8 #include "e2config.h"
9 #endif
10 #include "String.hpp"
11 #include "OptionContainer.hpp"
12 #include "NaughtyFilter.hpp"
13 #include "RegExp.hpp"
14 #include "ListContainer.hpp"
15 #include "Logger.hpp"
16
17 #include <cstring>
18 #include <algorithm>
19
20 // GLOBALS
21
22 extern OptionContainer o;
23
24 #ifdef HAVE_PCRE
25 extern RegExp absurl_re, relurl_re;
26 #endif
27
28 // DECLARATIONS
29
30 // category list entry class - stores category index & weight of all phrases
31 // found so far that fall under this category. also includes a less-than
32 // operator so that the STL sort algorithm can be applied to lists of these.
33 class listent
34 {
35 public:
36 listent()
37 : weight(0), string(""){};
38 listent(const int &w, String &s)
39 {
40 weight = w;
41 string = s;
42 };
43 int weight;
44 String string;
45 int operator<(const listent &a) const
46 {
47 // sort in descending order of score
48 return weight > a.weight ? 1 : 0;
49 };
50 };
51
52 // IMPLEMENTATION
53
54 // constructor - set up defaults
55 NaughtyFilter::NaughtyFilter()
56 {
57 ch_isiphost.comp(",*[a-z|A-Z].*");
58 }
59 NaughtyFilter::NaughtyFilter(HTTPHeader &request, HTTPHeader &response)
60 {
61 request_header = &request;
62 response_header = &response;
63 ch_isiphost.comp(",*[a-z|A-Z].*");
64 reset();
65 }
66
67 NaughtyFilter::NaughtyFilter(HTTPHeader &request, HTTPHeader &response, auth_rec &authrecin)
68 {
69 request_header = &request;
70 response_header = &response;
71 authrec = &authrecin;
72 ch_isiphost.comp(",*[a-z|A-Z].*");
73 reset();
74 }
75 void NaughtyFilter::setURL(bool set_ismitm) {
76 // do all of this normalisation etc just the once at the start.
77 url = request_header->getUrl(false, false);
78 baseurl = url;
79 baseurl.removeWhiteSpace();
80 baseurl.toLower();
81 baseurl.removePTP();
82 logurl = request_header->getLogUrl(false, set_ismitm);
83 urld = request_header->decode(url);
84 urldomain = url.getHostname();
85 urldomain.toLower();
86 String temp;
87 temp = request_header->getUrl(true,false); // get url with port
88 urldomainport = temp.getHostname();
89 urldomainport.toLower();
90
91 connect_site = urldomain;
92 isiphost = isIPHostnameStrip(urldomain);
93 is_ssl = request_header->requestType().startsWith("CONNECT");
94 ishead = request_header->requestType().startsWith("HEAD");
95 isconnect = is_ssl;
96 ismitm = set_ismitm;
97 if (baseurl == urldomain)
98 issiteonly = true;
99 docsize = 0;
100 }
101
102 void NaughtyFilter::setURL(String &sni) {
103 // do all of this normalisation etc just the once at the start.
104 url = sni;
105 baseurl = url;
106 baseurl.removeWhiteSpace();
107 baseurl.toLower();
108 baseurl.removePTP();
109 logurl = url;
110 urld = request_header->decode(url);
111 urldomain = url.getHostname();
112 urldomain.toLower();
113 connect_site = urldomain;
114 isiphost = isIPHostnameStrip(urldomain);
115 if (baseurl == urldomain)
116 issiteonly = true;
117 docsize = 0;
118 }
119
120 void NaughtyFilter::reset()
121 {
122 anon_user = o.log.anonymise_logs;
123 anon_url = false;
124 logurl = "";
125 isItNaughty = false;
126 isException = false;
127 whatIsNaughty = "";
128 whatIsNaughtyLog = "";
129 whatIsNaughtyCategories = "";
130 tempfilename = "";
131 tempfilemime= "";
132 tempfiledis= "";
133 usedisplaycats = false;
134 blocktype = 0;
135 store = false;
136 naughtiness = 0;
137 isGrey = false;
138 isSSLGrey = false;
139 isSearch = false;
140 message_no = 0;
141 is_text = false;
142 filtergroup = 0;
143 deep_urls_checked = false;
144 has_deep_urls = false;
145 issiteonly = false;
146
147 gomitm = false;
148 nomitm = false;
149 automitm = false;
150 // resets from CH
151 waschecked = false; // flags
152 wasrequested = false;
153 isexception = false;
154 issemiexception = false;
155 isourwebserver = false;
156 wasclean = false;
157 isbypass = false;
158 iscookiebypass = false;
159 isvirusbypass = false;
160 isbypassallowed = false;
161 isinfectionbypassallowed = false;
162 bypasstimestamp = 0;
163 isscanbypass = false;
164 ispostblock = false;
165 pausedtoobig = false;
166 wasinfected = false;
167 wasscanned = false;
168 contentmodified = false;
169 urlmodified = false;
170 headermodified = false;
171 headeradded = false;
172 nocheckcert = false;
173 noviruscheck = true;
174 headersent = 0;
175 message_no = 0;
176 log_message_no = 0;
177 urlredirect = false;
178 logcategory = false;
179 upfailure = false;
180 isdirect = o.net.no_proxy; // always set if no proxy defined
181 // authed = false;
182 // isbanneduser = false;
183 mimetype = "-";
184 docsize = 0; // to store the size of the returned document for logging
185 orig_ip = "";
186 orig_port = 0;
187 got_orig_ip = false;
188
189 }
190
191 // check the given document body for banned, weighted, and exception phrases (and PICS, and regexes, &c.)
192 // also used for scanning search terms, which causes various features - PICS, META/TITLE extraction, etc. - to be disabled
193 void NaughtyFilter::checkme(const char *rawbody, off_t rawbodylen, const String *url,
194 const String *domain, FOptionContainer* &foc, unsigned int phraselist, int limit, bool searchterms)
195 {
196 if (searchterms) {
197 DEBUG_content("Content flagged as search terms - disabling hex decoding, META/TITLE extraction & HTML removal");
198 }
199
200
201 if (foc->weighted_phrase_mode == 0) {
202 DEBUG_content("Weighted phrase mode 0 - not going any further.");
203 return;
204 }
205
206 // hex-decoded data (not case converted)
207 off_t hexdecodedlen = rawbodylen;
208 const char *hexdecoded = rawbody;
209
210 unsigned char c='\0';
211
212 // Hex decode content if desired
213 // Do this now, as it's not especially case-sensitive,
214 // and the case alteration should modify case post-decoding
215 // Search terms are already hex decoded, as they need to be to strip URL decoding
216 if (!searchterms && o.naughty.hex_decode_content) { // Mod suggested by AFN Tue 8th April 2003
217 DEBUG_content("Hex decoding is enabled");
218 char *hexdecoded_buf = new char[rawbodylen + 128 + 1];
219 memset(hexdecoded_buf, 0, rawbodylen + 128 + 1);
220 unsigned char c1;
221 unsigned char c2;
222 unsigned char c3;
223 char hexval[5] = "0x"; // Initializes a "hexadecimal string"
224 hexval[4] = '\0'; //TODO: should this not be hexval[2]??? - No this is correct but inefficient
225 char *ptr; // pointer required by strtol
226
227 // make a copy of the escaped document char by char
228 off_t i = 0;
229 off_t j = 0;
230 while (i < rawbodylen - 3) { // we lose 3 bytes but what the hell..
231 c1 = rawbody[i];
232 c2 = rawbody[i + 1];
233 c3 = rawbody[i + 2];
234 if (c1 == '%' && (((c2 >= '0') && (c2 <= '9')) || ((c2 >= 'a') && (c2 <= 'f')) || ((c2 >= 'A') && (c2 <= 'F')))
235 && (((c3 >= '0') && (c3 <= '9')) || ((c3 >= 'a') && (c3 <= 'f')) || ((c3 >= 'A') && (c3 <= 'F')))) {
236 hexval[2] = c2;
237 hexval[3] = c3;
238 c = (unsigned char)strtol(hexval, &ptr, 0);
239 i += 3;
240 } else {
241 c = c1;
242 i++;
243 }
244 hexdecoded_buf[j] = c;
245 j++;
246 }
247 // copy any remaining bytes
248 while (i < rawbodylen) {
249 hexdecoded_buf[j++] = rawbody[i++];
250 }
251 hexdecoded_buf[j] = '\0';
252 hexdecodedlen = j;
253 hexdecoded = hexdecoded_buf;
254 }
255
256 // scan twice, with & without case conversion (if desired) - aids support for exotic char encodings
257 // TODO: move META/title sentinel location outside this loop, as they are not case sensitive operations
258 bool preserve_case = (o.naughty.preserve_case != 0);
259 bool do_raw = false;
260 bool do_nohtml = false;
261
262 if (o.naughty.preserve_case == 2) {
263 // scanning twice *is* desired
264 // first time round the loop, don't preserve case (non-exotic encodings)
265 DEBUG_content("Filtering with/without case preservation is enabled");
266 preserve_case = false;
267 }
268
269 // Store for the lowercase (maybe) data
270 // The extra 128 is used for various speed tricks to
271 // squeeze as much speed as possible.
272 char *bodylc = new char[hexdecodedlen + 128 + 1];
273 memset(bodylc, 0, hexdecodedlen + 128 + 1);
274
275 // Store for the tag-stripped data
276 // Don't bother tag stripping search terms
277 char *bodynohtml = NULL;
278 if (!searchterms && (o.naughty.phrase_filter_mode == 1 || o.naughty.phrase_filter_mode == 2)) {
279 do_nohtml = true;
280 bodynohtml = new char[hexdecodedlen + 128 + 1];
281 memset(bodynohtml, 0, hexdecodedlen + 128 + 1);
282 }
283
284 if ((o.naughty.phrase_filter_mode == 0 || o.naughty.phrase_filter_mode == 2 || o.naughty.phrase_filter_mode == 3))
285 do_raw = true;
286
287 for (int loop = 0; loop < (o.naughty.preserve_case == 2 ? 2 : 1); loop++) {
288 DEBUG_content("Preserve case: ", preserve_case );
289
290 off_t i, j;
291 if (searchterms || do_raw)
292 DEBUG_content("Raw content needed");
293
294 // use the one that's been hex decoded, but not stripped
295 // make a copy of the document lowercase char by char
296 if (preserve_case) {
297 if (do_nohtml || o.naughty.phrase_filter_mode == 3) {
298 for (i = 0; i < hexdecodedlen; i++) {
299 c = hexdecoded[i];
300 // if (c == 13 || c == 9 || c == 10) {
301 if (c < '/' || c == ':' || c == ';' || c == '=' || c == '?' || c == '@' || (c > 90 && c < 97)) {
302 c = 32; // convert all whitespace and most punctuation marks to a space
303 }
304 bodylc[i] = c;
305 }
306 } else { // not being html striped so can remove < > now
307 for (i = 0; i < hexdecodedlen; i++) {
308 c = hexdecoded[i];
309 // if (c == 13 || c == 9 || c == 10) {
310 if (c < '/' || (c > '9' && c < 'A') || (c > 'Z' && c < 'a')) {
311 c = 32; // convert all whitespace and most punctuation marks to a space
312 }
313 bodylc[i] = c;
314 }
315 }
316 } else {
317 DEBUG_content("Not preserving case of raw content");
318 if (do_nohtml || o.naughty.phrase_filter_mode == 3) {
319 for (i = 0; i < hexdecodedlen; i++) {
320 c = hexdecoded[i];
321 if (c >= 'A' && c <= 'Z') {
322 c = 'a' + c - 'A';
323 } else if (c >= 192 && c <= 221) { // for accented chars
324 c += 32; // 224 + c - 192
325 } else {
326 //if (c == 13 || c == 9 || c == 10) {
327 if (c < '/' || c == ':' || c == ';' || c == '=' || c == '?' || c == '@' || (c > 90 && c < 97)) {
328 c = 32; // convert all whitespace and most punctuation marks to a space
329 }
330 }
331 bodylc[i] = c;
332 }
333 } else { // not being html striped so can remove < > now
334 for (i = 0; i < hexdecodedlen; i++) {
335 c = hexdecoded[i];
336 if (c >= 'A' && c <= 'Z') {
337 c = 'a' + c - 'A';
338 } else if (c >= 192 && c <= 221) { // for accented chars
339 c += 32; // 224 + c - 192
340 } else {
341 //if (c == 13 || c == 9 || c == 10) {
342 if (c < '/' || (c > '9' && c < 'A') || (c > 'Z' && c < 'a')) {
343 c = 32; // convert all whitespace and most punctuation marks to a space
344 }
345 }
346 bodylc[i] = c;
347 }
348 }
349 }
350
351 // filter meta tags & title only
352 // based on idea from Nicolas Peyrussie
353 if (!searchterms && (o.naughty.phrase_filter_mode == 3)) {
354 DEBUG_content("Filtering META/title");
355 bool addit = false; // flag if we should copy this char to filtered version
356 bool needcheck = false; // flag if we actually find anything worth filtering
357 off_t bodymetalen;
358
359 // find </head> or <body> as end of search range
360 char *endhead = strstr(bodylc, "</head");
361 if (endhead != NULL)
362 DEBUG_content("Found '</head', limiting search range");
363 if (endhead == NULL) {
364 endhead = strstr(bodylc, "<body");
365 if (endhead != NULL)
366 DEBUG_content("Found '<body', limiting search range");
367 }
368
369 // if case preserved, also look for uppercase versions
370 if (preserve_case and (endhead == NULL)) {
371 endhead = strstr(bodylc, "</HEAD");
372 if (endhead != NULL)
373 DEBUG_content("Found '</HEAD', limiting search range");
374 if (endhead == NULL) {
375 endhead = strstr(bodylc, "<BODY");
376 if (endhead != NULL)
377 DEBUG_content("Found '<BODY', limiting search range");
378 }
379 }
380
381 if (endhead == NULL)
382 endhead = bodylc + hexdecodedlen;
383
384 char *bodymeta = new char[(endhead - bodylc) + 128 + 1];
385 memset(bodymeta, 0, (endhead - bodylc) + 128 + 1);
386
387 // initialisation for removal of duplicate non-alphanumeric characters
388 j = 1;
389 bodymeta[0] = 32;
390
391 for (i = 0; i < (endhead - bodylc) - 7; i++) {
392 c = bodylc[i];
393 // are we at the start of a tag?
394 if ((!addit) && (c == '<')) {
395 if ((strncmp(bodylc + i + 1, "meta", 4) == 0) or (preserve_case and (strncmp(bodylc + i + 1, "META", 4) == 0))) {
396 DEBUG_content("Found META");
397 // start adding data to the check buffer
398 addit = true;
399 needcheck = true;
400 // skip 'meta '
401 i += 6;
402 c = bodylc[i];
403 }
404 // are we at the start of a title tag?
405 else if ((strncmp(bodylc + i + 1, "title", 5) == 0) or (preserve_case and (strncmp(bodylc + i + 1, "TITLE", 5) == 0))) {
406 DEBUG_content("Found TITLE");
407 // start adding data to the check buffer
408 addit = true;
409 needcheck = true;
410 // skip 'title>'
411 i += 7;
412 c = bodylc[i];
413 }
414 }
415 // meta tags end at a >
416 // title tags end at the next < (opening of </title>)
417 if (addit && ((c == '>') || (c == '<'))) {
418 // stop ading data
419 addit = false;
420 // add a space before the next word in the check buffer
421 bodymeta[j++] = 32;
422 }
423
424 if (addit) {
425 // if we're in "record" mode (i.e. inside a title/metatag), strip certain characters out
426 // of the data (to sanitise metatags & aid filtering of titles)
427 //if (c == ',' || c == '=' || c == '"' || c == '\''
428 // || c == '(' || c == ')' || c == '.') {
429 // // replace with a space
430 // c = 32;
431 //}
432 // don't bother duplicating spaces
433 if ((c != 32) || (c == 32 && (bodymeta[j - 1] != 32))) {
434 bodymeta[j++] = c; // copy it to the filtered copy
435 }
436 }
437 }
438 if (needcheck) {
439 bodymeta[j++] = '\0';
440 DEBUG_content(bodymeta);
441 bodymetalen = j;
442 checkphrase(bodymeta, bodymetalen, NULL, NULL, foc, phraselist, limit, searchterms);
443 }
444 else
445 DEBUG_content("Nothing to filter");
446
447 delete[] bodymeta;
448 // surely the intention is to search *only* meta/title, so always exit
449 delete[] bodylc;
450 delete[] bodynohtml;
451 if (hexdecoded != rawbody)
452 delete[] hexdecoded;
453 return;
454 }
455
456 if (do_nohtml) {
457 // if we fell through to here, use the one that's been hex decoded AND stripped
458 // Strip HTML
459 DEBUG_content("\"Smart\" filtering is enabled");
460 // we need this extra byte *
461 bool inhtml = false; // to flag if our pointer is within a html <>
462 bool addit; // flag if we should copy this char to filtered version
463 j = 1;
464 bodynohtml[0] = 32; // * for this
465 for (off_t i = 0; i < hexdecodedlen; i++) {
466 addit = true;
467 c = bodylc[i];
468 if (c == '<') {
469 inhtml = true; // flag we are inside a html <>
470 }
471 if (c == '>') { // flag we have just left a html <>
472 inhtml = false;
473 c = 32;
474 }
475 if (inhtml) {
476 addit = false;
477 }
478 if (c == 32) {
479 if (bodynohtml[j - 1] == 32) { // * and this
480 addit = false;
481 }
482 }
483 if (addit) { // if it passed the filters
484 bodynohtml[j++] = c; // copy it to the filtered copy
485 }
486 }
487 DEBUG_content("Checking smart content");
488 checkphrase(bodynohtml, j - 1, NULL, NULL, foc, phraselist, limit, searchterms);
489 if (isItNaughty || isException) {
490 delete[] bodylc;
491 delete[] bodynohtml;
492 if (hexdecoded != rawbody)
493 delete[] hexdecoded;
494 return; // Well there is no point in continuing is there?
495 }
496 }
497
498 if (!do_raw) {
499 delete[] bodylc;
500 delete[] bodynohtml;
501 if (hexdecoded != rawbody)
502 delete[] hexdecoded;
503 return; // only doing nohtml mode filtering
504 } else {
505 DEBUG_content("Checking raw content");
506
507 if (do_nohtml) { // already removed if not!
508 // replace html tag start and finish with space so that Start and finish words are detected
509 for (i = 0; i < hexdecodedlen; i++) {
510 c = bodylc[i];
511 if (c == '>' || c == '<')
512 bodylc[i] = 32;
513 }
514 }
515
516 // check unstripped content
517 checkphrase(bodylc, hexdecodedlen, url, domain, foc, phraselist, limit, searchterms);
518 if (isItNaughty || isException) {
519 delete[] bodylc;
520 delete[] bodynohtml;
521 if (hexdecoded != rawbody)
522 delete[] hexdecoded;
523 return; // Well there is no point in continuing is there?
524 }
525 }
526
527 // second time round the case loop (if there is a second time),
528 // do preserve case (exotic encodings)
529 preserve_case = true;
530 }
531 delete[] bodylc;
532 delete[] bodynohtml;
533 if (hexdecoded != rawbody)
534 delete[] hexdecoded;
535 }
536
537 // check the phrase lists
538 void NaughtyFilter::checkphrase(char *file, off_t filelen, const String *url, const String *domain,
539 FOptionContainer* &foc, unsigned int phraselist, int limit, bool searchterms)
540 {
541 int weighting = 0;
542 unsigned int cat;
543 RegResult Rre;
544 std::string weightedphrase;
545 String lastcategory;
546
547 // checkme: translate this?
548 String currcat("Embedded URLs");
549
550 // found categories list & reusable iterators
551 std::map<int, listent> listcategories;
552
553 // check for embedded references to banned sites/URLs.
554 // have regexes that check for URLs in pages (look for attributes (src, href, javascript location)
555 // or look for protocol strings (in which case, incl. ftp)?) and extract them.
556 // then check the extracted list against the banned site/URL lists.
557 // ADs category lists do not want to add to the possibility of a site being banned.
558 // Exception lists are not checked.
559 // Do not do full-blown category retrieval/duplicate checking; simply add the
560 // "Embedded URLs" category.
561 // Put a warning next to the option in the config file that this will take lots of CPU.
562 // Support phrase mode 1/2 distinction (duplicate sites/URLs).
563 // Have weight configurable per filter group, not globally or with a list directive -
564 // a weight of 0 will disable the option, effectively making this functionality per-FG itself.
565
566 // todo: if checkphrase is passed the domain & existing URL, it can create full URLs from relative ones.
567 // if a src/href URL starts with a /, append it to the domain; otherwise, append it to the existing URL.
568 // chop off anything after a ?, run through realPath, then put through the URL lists.
569
570 #ifdef HAVE_PCRE
571 // if weighted phrases are enabled, and we have been passed a URL and domain, and embedded URL checking is enabled...
572 // then check for embedded URLs!
573 #ifdef LEAVE_OUT_FOR_NOW
574 // TODO - THIS SECTION DISABLED IN V5 - REVIISIT LATER IF NEEDED
575 if (url != NULL && foc->embedded_url_weight > 0) {
576 std::map<int, listent>::iterator ourcat;
577 bool catinited = false;
578 std::map<String, unsigned int> found;
579 std::map<String, unsigned int>::iterator founditem;
580
581 String u;
582 char *j;
583
584 // check for absolute URLs
585 if (absurl_re.match(file, Rre)) {
586 // each match generates 2 results (because of the brackets in the regex),
587 // we're only interested in the first
588 DEBUG_content("Found ", Rre.numberOfMatches() / 2, " absolute URLs:");
589 for (int i = 0; i < Rre.numberOfMatches(); i += 2) {
590 // chop off quotes
591 u = Rre.result(i);
592 u = u.subString(1, u.length() - 2);
593 DEBUG_content(u);
594
595 if ((((j = foc->inBannedSiteList(u, false, false, false, lastcategory)) != NULL) &&
596 !(lastcategory.contains("ADs")))
597 || (((j = foc->inBannedURLList(u, false, false,false, lastcategory )) != NULL) &&
598 !(lastcategory.contains("ADs")))) {
599 // duplicate checking
600 // checkme: this should really be being done *before* we search the lists.
601 // but because inBanned* methods do some cleaning up of their own, we don't know the form to check against.
602 // we actually want these cleanups do be done before passing to inBanned*/inException* - this would
603 // speed up ConnectionHandler a bit too.
604 founditem = found.find(j);
605 if ((foc->weighted_phrase_mode == 2) && (founditem != found.end())) {
606 founditem->second++;
607 } else {
608 // add the site to the found phrases list
609 found[j] = 1;
610 if (weightedphrase.length() == 0)
611 weightedphrase = "[";
612 else
613 weightedphrase += " ";
614 weightedphrase += j;
615 if (!catinited) {
616 listcategories[-1] = listent(foc->embedded_url_weight, currcat);
617 ourcat = listcategories.find(-1);
618 catinited = true;
619 } else
620 ourcat->second.weight += foc->embedded_url_weight;
621 }
622 }
623 }
624 }
625
626 found.clear();
627
628 // check for relative URLs
629 if (relurl_re.match(file,Rre)) {
630 // we don't want any parameters on the end of the current URL, since we append to it directly
631 // when forming absolute URLs from relative ones. we do want a / on the end, too.
632 String currurl(*url);
633 if (currurl.contains("?"))
634 currurl = currurl.before("?");
635 if (currurl[currurl.length() - 1] != '/')
636 currurl += "/";
637
638 // each match generates 2 results (because of the brackets in the regex),
639 // we're only interested in the first
640 DEBUG_content("Found ", Rre.numberOfMatches() / 2, " relative URLs:");
641 for (int i = 0; i < Rre.numberOfMatches(); i += 2) {
642 u = Rre.result(i);
643
644 // can't find a way to negate submatches in PCRE, so it is entirely possible
645 // that some absolute URLs have made their way into this list. we don't want them.
646 if (u.contains("://"))
647 continue;
648
649 DEBUG_content(u);
650 // remove src/href & quotes
651 u = u.after("=");
652 u.removeWhiteSpace();
653 u = u.subString(1, u.length() - 2);
654
655 // create absolute URL
656 if (u[0] == '/')
657 u = (*domain) + u;
658 else
659 u = currurl + u;
660
661 DEBUG_content("absolute form: ", u );
662
663 if ((((j = foc->inBannedSiteList(u, false, false, false, lastcategory)) != NULL) &&
664 !(lastcategory.contains("ADs")))
665 || (((j = foc->inBannedURLList(u, false, false, false, lastcategory)) != NULL) &&
666 !(lastcategory.contains("ADs")))) {
667 // duplicate checking
668 // checkme: this should really be being done *before* we search the lists.
669 // but because inBanned* methods do some cleaning up of their own, we don't know the form to check against.
670 // we actually want these cleanups do be done before passing to inBanned*/inException* - this would
671 // speed up ConnectionHandler a bit too.
672 founditem = found.find(j);
673 if ((foc->weighted_phrase_mode == 2) && (founditem != found.end())) {
674 founditem->second++;
675 } else {
676 // add the site to the found phrases list
677 found[j] = 1;
678 if (weightedphrase.length() == 0)
679 weightedphrase = "[";
680 else
681 weightedphrase += " ";
682 weightedphrase += j;
683 if (!catinited) {
684 listcategories[-1] = listent(foc->embedded_url_weight, currcat);
685 ourcat = listcategories.find(-1);
686 catinited = true;
687 } else
688 ourcat->second.weight += foc->embedded_url_weight;
689 }
690 }
691 }
692 }
693 if (catinited) {
694 weighting = ourcat->second.weight;
695 weightedphrase += "]";
696 DEBUG_content(weightedphrase);
697 DEBUG_content("score from embedded URLs: ", ourcat->second.weight );
698 }
699 }
700 #endif // END OF LEAVE_OUT_FOR_NOW
701 #endif
702
703 std::string bannedphrase;
704 std::string exceptionphrase;
705 String bannedcategory;
706 int type, index, weight, time;
707 bool allcmatched = true, bannedcombi = false;
708 std::string s1;
709
710 // this line here searches for phrases contained in the list - the rest of the code is all sorting
711 // through it to find the categories, weightings, types etc. of what has actually been found.
712 std::map<std::string, std::pair<unsigned int, int> > found;
713 o.lm.l[phraselist]->graphSearch(found, file, filelen);
714
715 // cache reusable iterators
716 std::map<std::string, std::pair<unsigned int, int> >::iterator foundend = found.end();
717 std::map<std::string, std::pair<unsigned int, int> >::iterator foundcurrent;
718
719 // look for combinations first
720 //if banned must wait for exception later
721 std::string combifound;
722 std::string combisofar;
723
724 std::vector<int>::iterator combicurrent = o.lm.l[phraselist]->combilist.begin();
725 std::map<int, listent>::iterator catcurrent;
726 int lowest_occurrences = 0;
727
728 while (combicurrent != o.lm.l[phraselist]->combilist.end()) {
729 // Grab the current combination phrase part
730 index = *combicurrent;
731 // Do stuff if what we have is an end marker (end of one list of parts)
732 if (index == -2) {
733 // Were all the parts in this combination matched?
734 if (allcmatched) {
735 type = *(++combicurrent);
736 // check this time limit against the list of time limits
737 time = *(++combicurrent);
738 if (not(o.lm.l[phraselist]->checkTimeAtD(time))) {
739 // nope - so don't take any notice of it
740 #ifdef DEBUG_LOW
741 combicurrent++;
742 cat = (*++combicurrent);
743 DEBUG_content("Ignoring combi phrase based on time limits: ", combisofar, "; ",
744 o.lm.l[phraselist]->getListCategoryAtD(cat));
745 #else
746 combicurrent += 2;
747 #endif
748 combisofar = "";
749 } else if (type == -1) { // combination exception
750 isItNaughty = false;
751 isException = true;
752 // Combination exception phrase found:
753 // Combination exception search term found:
754 message_no = searchterms ? 456 : 605;
755 whatIsNaughtyLog = o.language_list.getTranslation(message_no);
756 whatIsNaughtyLog += combisofar;
757 whatIsNaughty = "";
758 ++combicurrent;
759 cat = *(++combicurrent);
760 whatIsNaughtyCategories = o.lm.l[phraselist]->getListCategoryAtD(cat);
761 return;
762 } else if (type == 1) { // combination weighting
763 weight = *(++combicurrent);
764 weighting += weight * (foc->weighted_phrase_mode == 2 ? 1 : lowest_occurrences);
765 if (weight > 0) {
766 cat = *(++combicurrent);
767 //category index -1 indicates an uncategorised list
768 if (cat >= 0) {
769 //don't output duplicate categories
770 catcurrent = listcategories.find(cat);
771 if (catcurrent != listcategories.end()) {
772 catcurrent->second.weight += weight * (foc->weighted_phrase_mode == 2 ? 1 : lowest_occurrences);
773 } else {
774 currcat = o.lm.l[phraselist]->getListCategoryAtD(cat);
775 listcategories[cat] = listent(weight, currcat);
776 }
777 }
778 } else {
779 // skip past category for negatively weighted phrases
780 combicurrent++;
781 }
782 if (weightedphrase.length() > 0) {
783 weightedphrase += "+";
784 }
785 weightedphrase += "(";
786 if (weight < 0) {
787 weightedphrase += "-" + combisofar;
788 } else {
789 weightedphrase += combisofar;
790 }
791
792 DEBUG_content("found combi weighted phrase (", foc->weighted_phrase_mode, "): ", combisofar,
793 " x", lowest_occurrences, " (per phrase: ", weight,
794 ", calculated: ", (weight * (foc->weighted_phrase_mode == 2 ? 1 : lowest_occurrences)), ")" );
795
796 weightedphrase += ")";
797 combisofar = "";
798 } else if (type == 0) { // combination banned
799 bannedcombi = true;
800 combifound += "(" + combisofar + ")";
801 combisofar = "";
802 combicurrent += 2;
803 cat = *(combicurrent);
804 bannedcategory = o.lm.l[phraselist]->getListCategoryAtD(cat);
805 }
806 } else {
807 // We had an end marker, but not all the parts so far were matched.
808 // Reset the match flag ready for the next chain, and advance to its first part.
809 allcmatched = true;
810 combicurrent += 4;
811 lowest_occurrences = 0;
812 }
813 } else {
814 // We didn't get an end marker - just an individual part.
815 // If all parts in the current chain have been matched so far, look for this one as well.
816 if (allcmatched) {
817 s1 = o.lm.l[phraselist]->getItemAtInt(index);
818 if ((foundcurrent = found.find(s1)) == foundend) {
819 allcmatched = false;
820 combisofar = "";
821 } else {
822 if (combisofar.length() > 0) {
823 combisofar += ", ";
824 }
825 combisofar += s1;
826 // also track lowest number of times any one part occurs in the text
827 // as this will correspond to the number of times the whole chain occurs
828 if ((lowest_occurrences == 0) || (lowest_occurrences > foundcurrent->second.second)) {
829 lowest_occurrences = foundcurrent->second.second;
830 }
831 }
832 }
833 }
834 // Advance to the next part in the current chain
835 combicurrent++;
836 }
837
838 // even if we already found a combi ban, we must still wait; there may be non-combi exceptions to follow
839
840 // now check non-combi phrases
841 foundcurrent = found.begin();
842 while (foundcurrent != foundend) {
843 // check time for current phrase
844 if (not o.lm.l[phraselist]->checkTimeAt(foundcurrent->second.first)) {
845 DEBUG_content("Ignoring phrase based on time limits: ", foundcurrent->first,
846 ", ", o.lm.l[phraselist]->getListCategoryAt(foundcurrent->second.first) );
847 foundcurrent++;
848 continue;
849 }
850 // 0=banned, 1=weighted, -1=exception, 2=combi, 3=weightedcombi
851 type = o.lm.l[phraselist]->getTypeAt(foundcurrent->second.first);
852 if (type == 0) {
853 // if we already found a combi ban, we don't need to know this stuff
854 if (!bannedcombi) {
855 isItNaughty = true;
856 bannedphrase = foundcurrent->first;
857 bannedcategory = o.lm.l[phraselist]->getListCategoryAt(foundcurrent->second.first, &cat);
858 }
859 } else if (type == 1) {
860 // found a weighted phrase - either add one lot of its score, or one lot for every occurrence, depending on phrase filtering mode
861 weight = o.lm.l[phraselist]->getWeightAt(foundcurrent->second.first) * (foc->weighted_phrase_mode == 2 ? 1 : foundcurrent->second.second);
862 weighting += weight;
863 if (weight > 0) {
864 currcat = o.lm.l[phraselist]->getListCategoryAt(foundcurrent->second.first, &cat);
865 if (cat >= 0) {
866 //don't output duplicate categories
867 catcurrent = listcategories.find(cat);
868 if (catcurrent != listcategories.end()) {
869 // add one or N times the weight to this category's score
870 catcurrent->second.weight += weight * (foc->weighted_phrase_mode == 2 ? 1 : foundcurrent->second.second);
871 } else {
872 listcategories[cat] = listent(weight, currcat);
873 }
874 }
875 }
876
877 if (o.naughty.show_weighted_found) {
878 if (weightedphrase.length() > 0) {
879 weightedphrase += "+";
880 }
881 if (weight < 0) {
882 weightedphrase += "-";
883 }
884
885 weightedphrase += foundcurrent->first;
886 }
887 DEBUG_content("found weighted phrase (", foc->weighted_phrase_mode, "): ", foundcurrent->first,
888 " x", foundcurrent->second.second,
889 " (per phrase: ", o.lm.l[phraselist]->getWeightAt(foundcurrent->second.first),
890 ", calculated: ", weight, ")" );
891 } else if (type == -1) {
892 isException = true;
893 isItNaughty = false;
894 // Exception phrase found:
895 // Exception search term found:
896 message_no = searchterms ? 457 : 604;
897 whatIsNaughtyLog = o.language_list.getTranslation(message_no);
898 whatIsNaughtyLog += foundcurrent->first;
899 whatIsNaughty = "";
900 whatIsNaughtyCategories = o.lm.l[phraselist]->getListCategoryAt(foundcurrent->second.first, NULL);
901 return; // no point in going further
902 }
903 foundcurrent++;
904 }
905
906 DEBUG_content("WEIGHTING: ", weighting );
907
908 // store the lowest negative weighting or highest positive weighting out of all filtering runs, preferring to store positive weightings.
909 if ((weighting < 0 && naughtiness <= 0 && weighting < naughtiness) || (naughtiness >= 0 && weighting > naughtiness) || (naughtiness < 0 && weighting > 0)) {
910 naughtiness = weighting;
911 }
912
913 DEBUG_content("NAUGHTINESS: ", naughtiness );
914
915 // *now* we can safely get down to the whole banning business!
916
917 if (bannedcombi) {
918 isItNaughty = true;
919 // Banned combination phrase found:
920 // Banned combination search term found:
921 message_no = searchterms ? 453 : 401;
922 log_message_no = searchterms ? 452 : 400;
923 whatIsNaughtyLog = o.language_list.getTranslation(log_message_no);
924 whatIsNaughtyLog += combifound;
925 // Banned combination phrase found.
926 // Banned combination search term found.
927 whatIsNaughty = o.language_list.getTranslation(message_no );
928 whatIsNaughtyCategories = bannedcategory.toCharArray();
929 return;
930 }
931
932 if (isItNaughty) {
933 // Banned phrase found:
934 // Banned search term found:
935 message_no = searchterms ? 450 : 300;
936 whatIsNaughtyLog = o.language_list.getTranslation(message_no);
937 whatIsNaughtyLog += bannedphrase;
938 // Banned phrase found.
939 // Banned search term found.
940 whatIsNaughty = o.language_list.getTranslation(searchterms ? 451 : 301);
941 whatIsNaughtyCategories = bannedcategory.toCharArray();
942 return;
943 }
944
945 bool log_it = o.naughty.show_all_weighted_found;
946 auto mno = searchterms ? 454 : 402;
947
948 if (weighting > limit) {
949 log_it = true;
950 isItNaughty = true;
951 message_no = mno;
952 whatIsNaughty = o.language_list.getTranslation(searchterms ? 455 : 403);
953 }
954
955 if(log_it) {
956 // Weighted phrase limit of
957 // Weighted search term limit of
958 whatIsNaughtyLog = o.language_list.getTranslation(mno);
959 whatIsNaughtyLog += String(limit).toCharArray();
960 whatIsNaughtyLog += " : ";
961 whatIsNaughtyLog += String(weighting).toCharArray();
962 if (o.naughty.show_weighted_found) {
963 whatIsNaughtyLog += " (";
964 whatIsNaughtyLog += weightedphrase;
965 whatIsNaughtyLog += ")";
966 }
967 // Weighted phrase limit exceeded.
968 // Weighted search term limit exceeded.
969 // Generate category list, sorted with highest scoring first.
970 bool nonempty = false;
971 bool belowthreshold = false;
972 String categories;
973 std::deque<listent> sortable_listcategories;
974 catcurrent = listcategories.begin();
975 while (catcurrent != listcategories.end()) {
976 sortable_listcategories.push_back(catcurrent->second);
977 catcurrent++;
978 }
979 std::sort(sortable_listcategories.begin(), sortable_listcategories.end());
980 std::deque<listent>::iterator k = sortable_listcategories.begin();
981 while (k != sortable_listcategories.end()) {
982 // if category display threshold is in use, apply it
983 if (!belowthreshold && (foc->category_threshold > 0)
984 && (k->weight < foc->category_threshold)) {
985 whatIsNaughtyDisplayCategories = categories.toCharArray();
986 belowthreshold = true;
987 usedisplaycats = true;
988 }
989 if (k->string.length() > 0) {
990 if (nonempty)
991 categories += ", ";
992 categories += k->string;
993 nonempty = true;
994 }
995 k++;
996 // if category threshold is set to show only the top category,
997 // everything after the first loop is below the threshold
998 if (!belowthreshold && foc->category_threshold < 0) {
999 whatIsNaughtyDisplayCategories = categories.toCharArray();
1000 belowthreshold = true;
1001 usedisplaycats = true;
1002 }
1003 }
1004 whatIsNaughtyCategories = categories.toCharArray();
1005 }
1006 return;
1007 // whatIsNaughty is what is displayed in the browser
1008 // whatIsNaughtyLog is what is logged in the log file if at all
1009 }
1010
1011
1012 // strip the URL down to just the IP/hostname, then do an isIPHostname on the result
1013 bool NaughtyFilter::isIPHostnameStrip(String url)
1014 {
1015 url = url.getHostname();
1016 if(ch_isiphost.match(url.toCharArray(), Rch_isiphost))
1017 return false;
1018 else
1019 return true;
1020 // return ldl->fg[0]->isIPHostname(url);
1021 }
1022
1023 String NaughtyFilter::getFlags() {
1024 String flags = listen_port;
1025 flags += ":";
1026 if (!(authrec == nullptr)) {
1027 if (authrec->is_transparent)
1028 flags += "T";
1029 else if (authrec->is_icap)
1030 flags += "I";
1031 else {
1032 if(authrec->is_tlsproxy)
1033 flags += "L";
1034 else
1035 flags += "P";
1036 }
1037
1038 if (o.log.addECHtoFlags && hasECH) {
1039 flags += "E";
1040 }
1041
1042 if (ismitm)
1043 flags += "M";
1044 else if (isconnect)
1045 flags += "S";
1046 else if (!authrec->is_icap)
1047 flags += "H";
1048
1049 flags += ":";
1050
1051 flags += authrec->user_source;
1052 flags += ":";
1053 flags += authrec->group_source;
1054 }
1055 return flags;
1056 }
1057
1058 String NaughtyFilter::get_lastmatch() {
1059 if (anon_url) {
1060 return lastmatch.anonimise();
1061 }
1062 return lastmatch;
1063 }
1064
1065 String NaughtyFilter::get_logUrl() {
1066 if (anon_url) {
1067 String t1 = logurl.before("://");
1068 String t2 = logurl.after("://");
1069 t1 += "://";
1070 t1 += t2.anonimise();
1071 return t1;
1072 }
1073 return logurl;
1074 }
1075
1076 String NaughtyFilter::main_category() {
1077 String temp = whatIsNaughtyCategories;
1078 if (temp.contains(",")) {
1079 temp = temp.before(",");
1080 }
1081 if (temp == "-") temp = "";
1082 temp.toLower();
1083 return temp;
1084 }
1085