| 1 |
// For all support, instructions and copyright go to: |
| 2 |
// http://e2guardian.org/ |
| 3 |
// Released under the GPL v2, with the OpenSSL exception described in the README file. |
| 4 |
|
| 5 |
// INCLUDES |
| 6 |
|
| 7 |
#ifdef HAVE_CONFIG_H |
| 8 |
#include "e2config.h" |
| 9 |
#endif |
| 10 |
#include "String.hpp" |
| 11 |
#include "OptionContainer.hpp" |
| 12 |
#include "NaughtyFilter.hpp" |
| 13 |
#include "RegExp.hpp" |
| 14 |
#include "ListContainer.hpp" |
| 15 |
#include "Logger.hpp" |
| 16 |
|
| 17 |
#include <cstring> |
| 18 |
#include <algorithm> |
| 19 |
|
| 20 |
// GLOBALS |
| 21 |
|
| 22 |
extern OptionContainer o; |
| 23 |
|
| 24 |
#ifdef HAVE_PCRE |
| 25 |
extern RegExp absurl_re, relurl_re; |
| 26 |
#endif |
| 27 |
|
| 28 |
// DECLARATIONS |
| 29 |
|
| 30 |
// category list entry class - stores category index & weight of all phrases |
| 31 |
// found so far that fall under this category. also includes a less-than |
| 32 |
// operator so that the STL sort algorithm can be applied to lists of these. |
| 33 |
class listent |
| 34 |
{ |
| 35 |
public: |
| 36 |
listent() |
| 37 |
: weight(0), string(""){}; |
| 38 |
listent(const int &w, String &s) |
| 39 |
{ |
| 40 |
weight = w; |
| 41 |
string = s; |
| 42 |
}; |
| 43 |
int weight; |
| 44 |
String string; |
| 45 |
int operator<(const listent &a) const |
| 46 |
{ |
| 47 |
// sort in descending order of score |
| 48 |
return weight > a.weight ? 1 : 0; |
| 49 |
}; |
| 50 |
}; |
| 51 |
|
| 52 |
// IMPLEMENTATION |
| 53 |
|
| 54 |
// constructor - set up defaults |
| 55 |
NaughtyFilter::NaughtyFilter() |
| 56 |
{ |
| 57 |
ch_isiphost.comp(",*[a-z|A-Z].*"); |
| 58 |
} |
| 59 |
NaughtyFilter::NaughtyFilter(HTTPHeader &request, HTTPHeader &response) |
| 60 |
{ |
| 61 |
request_header = &request; |
| 62 |
response_header = &response; |
| 63 |
ch_isiphost.comp(",*[a-z|A-Z].*"); |
| 64 |
reset(); |
| 65 |
} |
| 66 |
|
| 67 |
NaughtyFilter::NaughtyFilter(HTTPHeader &request, HTTPHeader &response, auth_rec &authrecin) |
| 68 |
{ |
| 69 |
request_header = &request; |
| 70 |
response_header = &response; |
| 71 |
authrec = &authrecin; |
| 72 |
ch_isiphost.comp(",*[a-z|A-Z].*"); |
| 73 |
reset(); |
| 74 |
} |
| 75 |
void NaughtyFilter::setURL(bool set_ismitm) { |
| 76 |
// do all of this normalisation etc just the once at the start. |
| 77 |
url = request_header->getUrl(false, false); |
| 78 |
baseurl = url; |
| 79 |
baseurl.removeWhiteSpace(); |
| 80 |
baseurl.toLower(); |
| 81 |
baseurl.removePTP(); |
| 82 |
logurl = request_header->getLogUrl(false, set_ismitm); |
| 83 |
urld = request_header->decode(url); |
| 84 |
urldomain = url.getHostname(); |
| 85 |
urldomain.toLower(); |
| 86 |
String temp; |
| 87 |
temp = request_header->getUrl(true,false); // get url with port |
| 88 |
urldomainport = temp.getHostname(); |
| 89 |
urldomainport.toLower(); |
| 90 |
|
| 91 |
connect_site = urldomain; |
| 92 |
isiphost = isIPHostnameStrip(urldomain); |
| 93 |
is_ssl = request_header->requestType().startsWith("CONNECT"); |
| 94 |
ishead = request_header->requestType().startsWith("HEAD"); |
| 95 |
isconnect = is_ssl; |
| 96 |
ismitm = set_ismitm; |
| 97 |
if (baseurl == urldomain) |
| 98 |
issiteonly = true; |
| 99 |
docsize = 0; |
| 100 |
} |
| 101 |
|
| 102 |
void NaughtyFilter::setURL(String &sni) { |
| 103 |
// do all of this normalisation etc just the once at the start. |
| 104 |
url = sni; |
| 105 |
baseurl = url; |
| 106 |
baseurl.removeWhiteSpace(); |
| 107 |
baseurl.toLower(); |
| 108 |
baseurl.removePTP(); |
| 109 |
logurl = url; |
| 110 |
urld = request_header->decode(url); |
| 111 |
urldomain = url.getHostname(); |
| 112 |
urldomain.toLower(); |
| 113 |
connect_site = urldomain; |
| 114 |
isiphost = isIPHostnameStrip(urldomain); |
| 115 |
if (baseurl == urldomain) |
| 116 |
issiteonly = true; |
| 117 |
docsize = 0; |
| 118 |
} |
| 119 |
|
| 120 |
void NaughtyFilter::reset() |
| 121 |
{ |
| 122 |
anon_user = o.log.anonymise_logs; |
| 123 |
anon_url = false; |
| 124 |
logurl = ""; |
| 125 |
isItNaughty = false; |
| 126 |
isException = false; |
| 127 |
whatIsNaughty = ""; |
| 128 |
whatIsNaughtyLog = ""; |
| 129 |
whatIsNaughtyCategories = ""; |
| 130 |
tempfilename = ""; |
| 131 |
tempfilemime= ""; |
| 132 |
tempfiledis= ""; |
| 133 |
usedisplaycats = false; |
| 134 |
blocktype = 0; |
| 135 |
store = false; |
| 136 |
naughtiness = 0; |
| 137 |
isGrey = false; |
| 138 |
isSSLGrey = false; |
| 139 |
isSearch = false; |
| 140 |
message_no = 0; |
| 141 |
is_text = false; |
| 142 |
filtergroup = 0; |
| 143 |
deep_urls_checked = false; |
| 144 |
has_deep_urls = false; |
| 145 |
issiteonly = false; |
| 146 |
|
| 147 |
gomitm = false; |
| 148 |
nomitm = false; |
| 149 |
automitm = false; |
| 150 |
// resets from CH |
| 151 |
waschecked = false; // flags |
| 152 |
wasrequested = false; |
| 153 |
isexception = false; |
| 154 |
issemiexception = false; |
| 155 |
isourwebserver = false; |
| 156 |
wasclean = false; |
| 157 |
isbypass = false; |
| 158 |
iscookiebypass = false; |
| 159 |
isvirusbypass = false; |
| 160 |
isbypassallowed = false; |
| 161 |
isinfectionbypassallowed = false; |
| 162 |
bypasstimestamp = 0; |
| 163 |
isscanbypass = false; |
| 164 |
ispostblock = false; |
| 165 |
pausedtoobig = false; |
| 166 |
wasinfected = false; |
| 167 |
wasscanned = false; |
| 168 |
contentmodified = false; |
| 169 |
urlmodified = false; |
| 170 |
headermodified = false; |
| 171 |
headeradded = false; |
| 172 |
nocheckcert = false; |
| 173 |
noviruscheck = true; |
| 174 |
headersent = 0; |
| 175 |
message_no = 0; |
| 176 |
log_message_no = 0; |
| 177 |
urlredirect = false; |
| 178 |
logcategory = false; |
| 179 |
upfailure = false; |
| 180 |
isdirect = o.net.no_proxy; // always set if no proxy defined |
| 181 |
// authed = false; |
| 182 |
// isbanneduser = false; |
| 183 |
mimetype = "-"; |
| 184 |
docsize = 0; // to store the size of the returned document for logging |
| 185 |
orig_ip = ""; |
| 186 |
orig_port = 0; |
| 187 |
got_orig_ip = false; |
| 188 |
|
| 189 |
} |
| 190 |
|
| 191 |
// check the given document body for banned, weighted, and exception phrases (and PICS, and regexes, &c.) |
| 192 |
// also used for scanning search terms, which causes various features - PICS, META/TITLE extraction, etc. - to be disabled |
| 193 |
void NaughtyFilter::checkme(const char *rawbody, off_t rawbodylen, const String *url, |
| 194 |
const String *domain, FOptionContainer* &foc, unsigned int phraselist, int limit, bool searchterms) |
| 195 |
{ |
| 196 |
if (searchterms) { |
| 197 |
DEBUG_content("Content flagged as search terms - disabling hex decoding, META/TITLE extraction & HTML removal"); |
| 198 |
} |
| 199 |
|
| 200 |
|
| 201 |
if (foc->weighted_phrase_mode == 0) { |
| 202 |
DEBUG_content("Weighted phrase mode 0 - not going any further."); |
| 203 |
return; |
| 204 |
} |
| 205 |
|
| 206 |
// hex-decoded data (not case converted) |
| 207 |
off_t hexdecodedlen = rawbodylen; |
| 208 |
const char *hexdecoded = rawbody; |
| 209 |
|
| 210 |
unsigned char c='\0'; |
| 211 |
|
| 212 |
// Hex decode content if desired |
| 213 |
// Do this now, as it's not especially case-sensitive, |
| 214 |
// and the case alteration should modify case post-decoding |
| 215 |
// Search terms are already hex decoded, as they need to be to strip URL decoding |
| 216 |
if (!searchterms && o.naughty.hex_decode_content) { // Mod suggested by AFN Tue 8th April 2003 |
| 217 |
DEBUG_content("Hex decoding is enabled"); |
| 218 |
char *hexdecoded_buf = new char[rawbodylen + 128 + 1]; |
| 219 |
memset(hexdecoded_buf, 0, rawbodylen + 128 + 1); |
| 220 |
unsigned char c1; |
| 221 |
unsigned char c2; |
| 222 |
unsigned char c3; |
| 223 |
char hexval[5] = "0x"; // Initializes a "hexadecimal string" |
| 224 |
hexval[4] = '\0'; //TODO: should this not be hexval[2]??? - No this is correct but inefficient |
| 225 |
char *ptr; // pointer required by strtol |
| 226 |
|
| 227 |
// make a copy of the escaped document char by char |
| 228 |
off_t i = 0; |
| 229 |
off_t j = 0; |
| 230 |
while (i < rawbodylen - 3) { // we lose 3 bytes but what the hell.. |
| 231 |
c1 = rawbody[i]; |
| 232 |
c2 = rawbody[i + 1]; |
| 233 |
c3 = rawbody[i + 2]; |
| 234 |
if (c1 == '%' && (((c2 >= '0') && (c2 <= '9')) || ((c2 >= 'a') && (c2 <= 'f')) || ((c2 >= 'A') && (c2 <= 'F'))) |
| 235 |
&& (((c3 >= '0') && (c3 <= '9')) || ((c3 >= 'a') && (c3 <= 'f')) || ((c3 >= 'A') && (c3 <= 'F')))) { |
| 236 |
hexval[2] = c2; |
| 237 |
hexval[3] = c3; |
| 238 |
c = (unsigned char)strtol(hexval, &ptr, 0); |
| 239 |
i += 3; |
| 240 |
} else { |
| 241 |
c = c1; |
| 242 |
i++; |
| 243 |
} |
| 244 |
hexdecoded_buf[j] = c; |
| 245 |
j++; |
| 246 |
} |
| 247 |
// copy any remaining bytes |
| 248 |
while (i < rawbodylen) { |
| 249 |
hexdecoded_buf[j++] = rawbody[i++]; |
| 250 |
} |
| 251 |
hexdecoded_buf[j] = '\0'; |
| 252 |
hexdecodedlen = j; |
| 253 |
hexdecoded = hexdecoded_buf; |
| 254 |
} |
| 255 |
|
| 256 |
// scan twice, with & without case conversion (if desired) - aids support for exotic char encodings |
| 257 |
// TODO: move META/title sentinel location outside this loop, as they are not case sensitive operations |
| 258 |
bool preserve_case = (o.naughty.preserve_case != 0); |
| 259 |
bool do_raw = false; |
| 260 |
bool do_nohtml = false; |
| 261 |
|
| 262 |
if (o.naughty.preserve_case == 2) { |
| 263 |
// scanning twice *is* desired |
| 264 |
// first time round the loop, don't preserve case (non-exotic encodings) |
| 265 |
DEBUG_content("Filtering with/without case preservation is enabled"); |
| 266 |
preserve_case = false; |
| 267 |
} |
| 268 |
|
| 269 |
// Store for the lowercase (maybe) data |
| 270 |
// The extra 128 is used for various speed tricks to |
| 271 |
// squeeze as much speed as possible. |
| 272 |
char *bodylc = new char[hexdecodedlen + 128 + 1]; |
| 273 |
memset(bodylc, 0, hexdecodedlen + 128 + 1); |
| 274 |
|
| 275 |
// Store for the tag-stripped data |
| 276 |
// Don't bother tag stripping search terms |
| 277 |
char *bodynohtml = NULL; |
| 278 |
if (!searchterms && (o.naughty.phrase_filter_mode == 1 || o.naughty.phrase_filter_mode == 2)) { |
| 279 |
do_nohtml = true; |
| 280 |
bodynohtml = new char[hexdecodedlen + 128 + 1]; |
| 281 |
memset(bodynohtml, 0, hexdecodedlen + 128 + 1); |
| 282 |
} |
| 283 |
|
| 284 |
if ((o.naughty.phrase_filter_mode == 0 || o.naughty.phrase_filter_mode == 2 || o.naughty.phrase_filter_mode == 3)) |
| 285 |
do_raw = true; |
| 286 |
|
| 287 |
for (int loop = 0; loop < (o.naughty.preserve_case == 2 ? 2 : 1); loop++) { |
| 288 |
DEBUG_content("Preserve case: ", preserve_case ); |
| 289 |
|
| 290 |
off_t i, j; |
| 291 |
if (searchterms || do_raw) |
| 292 |
DEBUG_content("Raw content needed"); |
| 293 |
|
| 294 |
// use the one that's been hex decoded, but not stripped |
| 295 |
// make a copy of the document lowercase char by char |
| 296 |
if (preserve_case) { |
| 297 |
if (do_nohtml || o.naughty.phrase_filter_mode == 3) { |
| 298 |
for (i = 0; i < hexdecodedlen; i++) { |
| 299 |
c = hexdecoded[i]; |
| 300 |
// if (c == 13 || c == 9 || c == 10) { |
| 301 |
if (c < '/' || c == ':' || c == ';' || c == '=' || c == '?' || c == '@' || (c > 90 && c < 97)) { |
| 302 |
c = 32; // convert all whitespace and most punctuation marks to a space |
| 303 |
} |
| 304 |
bodylc[i] = c; |
| 305 |
} |
| 306 |
} else { // not being html striped so can remove < > now |
| 307 |
for (i = 0; i < hexdecodedlen; i++) { |
| 308 |
c = hexdecoded[i]; |
| 309 |
// if (c == 13 || c == 9 || c == 10) { |
| 310 |
if (c < '/' || (c > '9' && c < 'A') || (c > 'Z' && c < 'a')) { |
| 311 |
c = 32; // convert all whitespace and most punctuation marks to a space |
| 312 |
} |
| 313 |
bodylc[i] = c; |
| 314 |
} |
| 315 |
} |
| 316 |
} else { |
| 317 |
DEBUG_content("Not preserving case of raw content"); |
| 318 |
if (do_nohtml || o.naughty.phrase_filter_mode == 3) { |
| 319 |
for (i = 0; i < hexdecodedlen; i++) { |
| 320 |
c = hexdecoded[i]; |
| 321 |
if (c >= 'A' && c <= 'Z') { |
| 322 |
c = 'a' + c - 'A'; |
| 323 |
} else if (c >= 192 && c <= 221) { // for accented chars |
| 324 |
c += 32; // 224 + c - 192 |
| 325 |
} else { |
| 326 |
//if (c == 13 || c == 9 || c == 10) { |
| 327 |
if (c < '/' || c == ':' || c == ';' || c == '=' || c == '?' || c == '@' || (c > 90 && c < 97)) { |
| 328 |
c = 32; // convert all whitespace and most punctuation marks to a space |
| 329 |
} |
| 330 |
} |
| 331 |
bodylc[i] = c; |
| 332 |
} |
| 333 |
} else { // not being html striped so can remove < > now |
| 334 |
for (i = 0; i < hexdecodedlen; i++) { |
| 335 |
c = hexdecoded[i]; |
| 336 |
if (c >= 'A' && c <= 'Z') { |
| 337 |
c = 'a' + c - 'A'; |
| 338 |
} else if (c >= 192 && c <= 221) { // for accented chars |
| 339 |
c += 32; // 224 + c - 192 |
| 340 |
} else { |
| 341 |
//if (c == 13 || c == 9 || c == 10) { |
| 342 |
if (c < '/' || (c > '9' && c < 'A') || (c > 'Z' && c < 'a')) { |
| 343 |
c = 32; // convert all whitespace and most punctuation marks to a space |
| 344 |
} |
| 345 |
} |
| 346 |
bodylc[i] = c; |
| 347 |
} |
| 348 |
} |
| 349 |
} |
| 350 |
|
| 351 |
// filter meta tags & title only |
| 352 |
// based on idea from Nicolas Peyrussie |
| 353 |
if (!searchterms && (o.naughty.phrase_filter_mode == 3)) { |
| 354 |
DEBUG_content("Filtering META/title"); |
| 355 |
bool addit = false; // flag if we should copy this char to filtered version |
| 356 |
bool needcheck = false; // flag if we actually find anything worth filtering |
| 357 |
off_t bodymetalen; |
| 358 |
|
| 359 |
// find </head> or <body> as end of search range |
| 360 |
char *endhead = strstr(bodylc, "</head"); |
| 361 |
if (endhead != NULL) |
| 362 |
DEBUG_content("Found '</head', limiting search range"); |
| 363 |
if (endhead == NULL) { |
| 364 |
endhead = strstr(bodylc, "<body"); |
| 365 |
if (endhead != NULL) |
| 366 |
DEBUG_content("Found '<body', limiting search range"); |
| 367 |
} |
| 368 |
|
| 369 |
// if case preserved, also look for uppercase versions |
| 370 |
if (preserve_case and (endhead == NULL)) { |
| 371 |
endhead = strstr(bodylc, "</HEAD"); |
| 372 |
if (endhead != NULL) |
| 373 |
DEBUG_content("Found '</HEAD', limiting search range"); |
| 374 |
if (endhead == NULL) { |
| 375 |
endhead = strstr(bodylc, "<BODY"); |
| 376 |
if (endhead != NULL) |
| 377 |
DEBUG_content("Found '<BODY', limiting search range"); |
| 378 |
} |
| 379 |
} |
| 380 |
|
| 381 |
if (endhead == NULL) |
| 382 |
endhead = bodylc + hexdecodedlen; |
| 383 |
|
| 384 |
char *bodymeta = new char[(endhead - bodylc) + 128 + 1]; |
| 385 |
memset(bodymeta, 0, (endhead - bodylc) + 128 + 1); |
| 386 |
|
| 387 |
// initialisation for removal of duplicate non-alphanumeric characters |
| 388 |
j = 1; |
| 389 |
bodymeta[0] = 32; |
| 390 |
|
| 391 |
for (i = 0; i < (endhead - bodylc) - 7; i++) { |
| 392 |
c = bodylc[i]; |
| 393 |
// are we at the start of a tag? |
| 394 |
if ((!addit) && (c == '<')) { |
| 395 |
if ((strncmp(bodylc + i + 1, "meta", 4) == 0) or (preserve_case and (strncmp(bodylc + i + 1, "META", 4) == 0))) { |
| 396 |
DEBUG_content("Found META"); |
| 397 |
// start adding data to the check buffer |
| 398 |
addit = true; |
| 399 |
needcheck = true; |
| 400 |
// skip 'meta ' |
| 401 |
i += 6; |
| 402 |
c = bodylc[i]; |
| 403 |
} |
| 404 |
// are we at the start of a title tag? |
| 405 |
else if ((strncmp(bodylc + i + 1, "title", 5) == 0) or (preserve_case and (strncmp(bodylc + i + 1, "TITLE", 5) == 0))) { |
| 406 |
DEBUG_content("Found TITLE"); |
| 407 |
// start adding data to the check buffer |
| 408 |
addit = true; |
| 409 |
needcheck = true; |
| 410 |
// skip 'title>' |
| 411 |
i += 7; |
| 412 |
c = bodylc[i]; |
| 413 |
} |
| 414 |
} |
| 415 |
// meta tags end at a > |
| 416 |
// title tags end at the next < (opening of </title>) |
| 417 |
if (addit && ((c == '>') || (c == '<'))) { |
| 418 |
// stop ading data |
| 419 |
addit = false; |
| 420 |
// add a space before the next word in the check buffer |
| 421 |
bodymeta[j++] = 32; |
| 422 |
} |
| 423 |
|
| 424 |
if (addit) { |
| 425 |
// if we're in "record" mode (i.e. inside a title/metatag), strip certain characters out |
| 426 |
// of the data (to sanitise metatags & aid filtering of titles) |
| 427 |
//if (c == ',' || c == '=' || c == '"' || c == '\'' |
| 428 |
// || c == '(' || c == ')' || c == '.') { |
| 429 |
// // replace with a space |
| 430 |
// c = 32; |
| 431 |
//} |
| 432 |
// don't bother duplicating spaces |
| 433 |
if ((c != 32) || (c == 32 && (bodymeta[j - 1] != 32))) { |
| 434 |
bodymeta[j++] = c; // copy it to the filtered copy |
| 435 |
} |
| 436 |
} |
| 437 |
} |
| 438 |
if (needcheck) { |
| 439 |
bodymeta[j++] = '\0'; |
| 440 |
DEBUG_content(bodymeta); |
| 441 |
bodymetalen = j; |
| 442 |
checkphrase(bodymeta, bodymetalen, NULL, NULL, foc, phraselist, limit, searchterms); |
| 443 |
} |
| 444 |
else |
| 445 |
DEBUG_content("Nothing to filter"); |
| 446 |
|
| 447 |
delete[] bodymeta; |
| 448 |
// surely the intention is to search *only* meta/title, so always exit |
| 449 |
delete[] bodylc; |
| 450 |
delete[] bodynohtml; |
| 451 |
if (hexdecoded != rawbody) |
| 452 |
delete[] hexdecoded; |
| 453 |
return; |
| 454 |
} |
| 455 |
|
| 456 |
if (do_nohtml) { |
| 457 |
// if we fell through to here, use the one that's been hex decoded AND stripped |
| 458 |
// Strip HTML |
| 459 |
DEBUG_content("\"Smart\" filtering is enabled"); |
| 460 |
// we need this extra byte * |
| 461 |
bool inhtml = false; // to flag if our pointer is within a html <> |
| 462 |
bool addit; // flag if we should copy this char to filtered version |
| 463 |
j = 1; |
| 464 |
bodynohtml[0] = 32; // * for this |
| 465 |
for (off_t i = 0; i < hexdecodedlen; i++) { |
| 466 |
addit = true; |
| 467 |
c = bodylc[i]; |
| 468 |
if (c == '<') { |
| 469 |
inhtml = true; // flag we are inside a html <> |
| 470 |
} |
| 471 |
if (c == '>') { // flag we have just left a html <> |
| 472 |
inhtml = false; |
| 473 |
c = 32; |
| 474 |
} |
| 475 |
if (inhtml) { |
| 476 |
addit = false; |
| 477 |
} |
| 478 |
if (c == 32) { |
| 479 |
if (bodynohtml[j - 1] == 32) { // * and this |
| 480 |
addit = false; |
| 481 |
} |
| 482 |
} |
| 483 |
if (addit) { // if it passed the filters |
| 484 |
bodynohtml[j++] = c; // copy it to the filtered copy |
| 485 |
} |
| 486 |
} |
| 487 |
DEBUG_content("Checking smart content"); |
| 488 |
checkphrase(bodynohtml, j - 1, NULL, NULL, foc, phraselist, limit, searchterms); |
| 489 |
if (isItNaughty || isException) { |
| 490 |
delete[] bodylc; |
| 491 |
delete[] bodynohtml; |
| 492 |
if (hexdecoded != rawbody) |
| 493 |
delete[] hexdecoded; |
| 494 |
return; // Well there is no point in continuing is there? |
| 495 |
} |
| 496 |
} |
| 497 |
|
| 498 |
if (!do_raw) { |
| 499 |
delete[] bodylc; |
| 500 |
delete[] bodynohtml; |
| 501 |
if (hexdecoded != rawbody) |
| 502 |
delete[] hexdecoded; |
| 503 |
return; // only doing nohtml mode filtering |
| 504 |
} else { |
| 505 |
DEBUG_content("Checking raw content"); |
| 506 |
|
| 507 |
if (do_nohtml) { // already removed if not! |
| 508 |
// replace html tag start and finish with space so that Start and finish words are detected |
| 509 |
for (i = 0; i < hexdecodedlen; i++) { |
| 510 |
c = bodylc[i]; |
| 511 |
if (c == '>' || c == '<') |
| 512 |
bodylc[i] = 32; |
| 513 |
} |
| 514 |
} |
| 515 |
|
| 516 |
// check unstripped content |
| 517 |
checkphrase(bodylc, hexdecodedlen, url, domain, foc, phraselist, limit, searchterms); |
| 518 |
if (isItNaughty || isException) { |
| 519 |
delete[] bodylc; |
| 520 |
delete[] bodynohtml; |
| 521 |
if (hexdecoded != rawbody) |
| 522 |
delete[] hexdecoded; |
| 523 |
return; // Well there is no point in continuing is there? |
| 524 |
} |
| 525 |
} |
| 526 |
|
| 527 |
// second time round the case loop (if there is a second time), |
| 528 |
// do preserve case (exotic encodings) |
| 529 |
preserve_case = true; |
| 530 |
} |
| 531 |
delete[] bodylc; |
| 532 |
delete[] bodynohtml; |
| 533 |
if (hexdecoded != rawbody) |
| 534 |
delete[] hexdecoded; |
| 535 |
} |
| 536 |
|
| 537 |
// check the phrase lists |
| 538 |
void NaughtyFilter::checkphrase(char *file, off_t filelen, const String *url, const String *domain, |
| 539 |
FOptionContainer* &foc, unsigned int phraselist, int limit, bool searchterms) |
| 540 |
{ |
| 541 |
int weighting = 0; |
| 542 |
unsigned int cat; |
| 543 |
RegResult Rre; |
| 544 |
std::string weightedphrase; |
| 545 |
String lastcategory; |
| 546 |
|
| 547 |
// checkme: translate this? |
| 548 |
String currcat("Embedded URLs"); |
| 549 |
|
| 550 |
// found categories list & reusable iterators |
| 551 |
std::map<int, listent> listcategories; |
| 552 |
|
| 553 |
// check for embedded references to banned sites/URLs. |
| 554 |
// have regexes that check for URLs in pages (look for attributes (src, href, javascript location) |
| 555 |
// or look for protocol strings (in which case, incl. ftp)?) and extract them. |
| 556 |
// then check the extracted list against the banned site/URL lists. |
| 557 |
// ADs category lists do not want to add to the possibility of a site being banned. |
| 558 |
// Exception lists are not checked. |
| 559 |
// Do not do full-blown category retrieval/duplicate checking; simply add the |
| 560 |
// "Embedded URLs" category. |
| 561 |
// Put a warning next to the option in the config file that this will take lots of CPU. |
| 562 |
// Support phrase mode 1/2 distinction (duplicate sites/URLs). |
| 563 |
// Have weight configurable per filter group, not globally or with a list directive - |
| 564 |
// a weight of 0 will disable the option, effectively making this functionality per-FG itself. |
| 565 |
|
| 566 |
// todo: if checkphrase is passed the domain & existing URL, it can create full URLs from relative ones. |
| 567 |
// if a src/href URL starts with a /, append it to the domain; otherwise, append it to the existing URL. |
| 568 |
// chop off anything after a ?, run through realPath, then put through the URL lists. |
| 569 |
|
| 570 |
#ifdef HAVE_PCRE |
| 571 |
// if weighted phrases are enabled, and we have been passed a URL and domain, and embedded URL checking is enabled... |
| 572 |
// then check for embedded URLs! |
| 573 |
#ifdef LEAVE_OUT_FOR_NOW |
| 574 |
// TODO - THIS SECTION DISABLED IN V5 - REVIISIT LATER IF NEEDED |
| 575 |
if (url != NULL && foc->embedded_url_weight > 0) { |
| 576 |
std::map<int, listent>::iterator ourcat; |
| 577 |
bool catinited = false; |
| 578 |
std::map<String, unsigned int> found; |
| 579 |
std::map<String, unsigned int>::iterator founditem; |
| 580 |
|
| 581 |
String u; |
| 582 |
char *j; |
| 583 |
|
| 584 |
// check for absolute URLs |
| 585 |
if (absurl_re.match(file, Rre)) { |
| 586 |
// each match generates 2 results (because of the brackets in the regex), |
| 587 |
// we're only interested in the first |
| 588 |
DEBUG_content("Found ", Rre.numberOfMatches() / 2, " absolute URLs:"); |
| 589 |
for (int i = 0; i < Rre.numberOfMatches(); i += 2) { |
| 590 |
// chop off quotes |
| 591 |
u = Rre.result(i); |
| 592 |
u = u.subString(1, u.length() - 2); |
| 593 |
DEBUG_content(u); |
| 594 |
|
| 595 |
if ((((j = foc->inBannedSiteList(u, false, false, false, lastcategory)) != NULL) && |
| 596 |
!(lastcategory.contains("ADs"))) |
| 597 |
|| (((j = foc->inBannedURLList(u, false, false,false, lastcategory )) != NULL) && |
| 598 |
!(lastcategory.contains("ADs")))) { |
| 599 |
// duplicate checking |
| 600 |
// checkme: this should really be being done *before* we search the lists. |
| 601 |
// but because inBanned* methods do some cleaning up of their own, we don't know the form to check against. |
| 602 |
// we actually want these cleanups do be done before passing to inBanned*/inException* - this would |
| 603 |
// speed up ConnectionHandler a bit too. |
| 604 |
founditem = found.find(j); |
| 605 |
if ((foc->weighted_phrase_mode == 2) && (founditem != found.end())) { |
| 606 |
founditem->second++; |
| 607 |
} else { |
| 608 |
// add the site to the found phrases list |
| 609 |
found[j] = 1; |
| 610 |
if (weightedphrase.length() == 0) |
| 611 |
weightedphrase = "["; |
| 612 |
else |
| 613 |
weightedphrase += " "; |
| 614 |
weightedphrase += j; |
| 615 |
if (!catinited) { |
| 616 |
listcategories[-1] = listent(foc->embedded_url_weight, currcat); |
| 617 |
ourcat = listcategories.find(-1); |
| 618 |
catinited = true; |
| 619 |
} else |
| 620 |
ourcat->second.weight += foc->embedded_url_weight; |
| 621 |
} |
| 622 |
} |
| 623 |
} |
| 624 |
} |
| 625 |
|
| 626 |
found.clear(); |
| 627 |
|
| 628 |
// check for relative URLs |
| 629 |
if (relurl_re.match(file,Rre)) { |
| 630 |
// we don't want any parameters on the end of the current URL, since we append to it directly |
| 631 |
// when forming absolute URLs from relative ones. we do want a / on the end, too. |
| 632 |
String currurl(*url); |
| 633 |
if (currurl.contains("?")) |
| 634 |
currurl = currurl.before("?"); |
| 635 |
if (currurl[currurl.length() - 1] != '/') |
| 636 |
currurl += "/"; |
| 637 |
|
| 638 |
// each match generates 2 results (because of the brackets in the regex), |
| 639 |
// we're only interested in the first |
| 640 |
DEBUG_content("Found ", Rre.numberOfMatches() / 2, " relative URLs:"); |
| 641 |
for (int i = 0; i < Rre.numberOfMatches(); i += 2) { |
| 642 |
u = Rre.result(i); |
| 643 |
|
| 644 |
// can't find a way to negate submatches in PCRE, so it is entirely possible |
| 645 |
// that some absolute URLs have made their way into this list. we don't want them. |
| 646 |
if (u.contains("://")) |
| 647 |
continue; |
| 648 |
|
| 649 |
DEBUG_content(u); |
| 650 |
// remove src/href & quotes |
| 651 |
u = u.after("="); |
| 652 |
u.removeWhiteSpace(); |
| 653 |
u = u.subString(1, u.length() - 2); |
| 654 |
|
| 655 |
// create absolute URL |
| 656 |
if (u[0] == '/') |
| 657 |
u = (*domain) + u; |
| 658 |
else |
| 659 |
u = currurl + u; |
| 660 |
|
| 661 |
DEBUG_content("absolute form: ", u ); |
| 662 |
|
| 663 |
if ((((j = foc->inBannedSiteList(u, false, false, false, lastcategory)) != NULL) && |
| 664 |
!(lastcategory.contains("ADs"))) |
| 665 |
|| (((j = foc->inBannedURLList(u, false, false, false, lastcategory)) != NULL) && |
| 666 |
!(lastcategory.contains("ADs")))) { |
| 667 |
// duplicate checking |
| 668 |
// checkme: this should really be being done *before* we search the lists. |
| 669 |
// but because inBanned* methods do some cleaning up of their own, we don't know the form to check against. |
| 670 |
// we actually want these cleanups do be done before passing to inBanned*/inException* - this would |
| 671 |
// speed up ConnectionHandler a bit too. |
| 672 |
founditem = found.find(j); |
| 673 |
if ((foc->weighted_phrase_mode == 2) && (founditem != found.end())) { |
| 674 |
founditem->second++; |
| 675 |
} else { |
| 676 |
// add the site to the found phrases list |
| 677 |
found[j] = 1; |
| 678 |
if (weightedphrase.length() == 0) |
| 679 |
weightedphrase = "["; |
| 680 |
else |
| 681 |
weightedphrase += " "; |
| 682 |
weightedphrase += j; |
| 683 |
if (!catinited) { |
| 684 |
listcategories[-1] = listent(foc->embedded_url_weight, currcat); |
| 685 |
ourcat = listcategories.find(-1); |
| 686 |
catinited = true; |
| 687 |
} else |
| 688 |
ourcat->second.weight += foc->embedded_url_weight; |
| 689 |
} |
| 690 |
} |
| 691 |
} |
| 692 |
} |
| 693 |
if (catinited) { |
| 694 |
weighting = ourcat->second.weight; |
| 695 |
weightedphrase += "]"; |
| 696 |
DEBUG_content(weightedphrase); |
| 697 |
DEBUG_content("score from embedded URLs: ", ourcat->second.weight ); |
| 698 |
} |
| 699 |
} |
| 700 |
#endif // END OF LEAVE_OUT_FOR_NOW |
| 701 |
#endif |
| 702 |
|
| 703 |
std::string bannedphrase; |
| 704 |
std::string exceptionphrase; |
| 705 |
String bannedcategory; |
| 706 |
int type, index, weight, time; |
| 707 |
bool allcmatched = true, bannedcombi = false; |
| 708 |
std::string s1; |
| 709 |
|
| 710 |
// this line here searches for phrases contained in the list - the rest of the code is all sorting |
| 711 |
// through it to find the categories, weightings, types etc. of what has actually been found. |
| 712 |
std::map<std::string, std::pair<unsigned int, int> > found; |
| 713 |
o.lm.l[phraselist]->graphSearch(found, file, filelen); |
| 714 |
|
| 715 |
// cache reusable iterators |
| 716 |
std::map<std::string, std::pair<unsigned int, int> >::iterator foundend = found.end(); |
| 717 |
std::map<std::string, std::pair<unsigned int, int> >::iterator foundcurrent; |
| 718 |
|
| 719 |
// look for combinations first |
| 720 |
//if banned must wait for exception later |
| 721 |
std::string combifound; |
| 722 |
std::string combisofar; |
| 723 |
|
| 724 |
std::vector<int>::iterator combicurrent = o.lm.l[phraselist]->combilist.begin(); |
| 725 |
std::map<int, listent>::iterator catcurrent; |
| 726 |
int lowest_occurrences = 0; |
| 727 |
|
| 728 |
while (combicurrent != o.lm.l[phraselist]->combilist.end()) { |
| 729 |
// Grab the current combination phrase part |
| 730 |
index = *combicurrent; |
| 731 |
// Do stuff if what we have is an end marker (end of one list of parts) |
| 732 |
if (index == -2) { |
| 733 |
// Were all the parts in this combination matched? |
| 734 |
if (allcmatched) { |
| 735 |
type = *(++combicurrent); |
| 736 |
// check this time limit against the list of time limits |
| 737 |
time = *(++combicurrent); |
| 738 |
if (not(o.lm.l[phraselist]->checkTimeAtD(time))) { |
| 739 |
// nope - so don't take any notice of it |
| 740 |
#ifdef DEBUG_LOW |
| 741 |
combicurrent++; |
| 742 |
cat = (*++combicurrent); |
| 743 |
DEBUG_content("Ignoring combi phrase based on time limits: ", combisofar, "; ", |
| 744 |
o.lm.l[phraselist]->getListCategoryAtD(cat)); |
| 745 |
#else |
| 746 |
combicurrent += 2; |
| 747 |
#endif |
| 748 |
combisofar = ""; |
| 749 |
} else if (type == -1) { // combination exception |
| 750 |
isItNaughty = false; |
| 751 |
isException = true; |
| 752 |
// Combination exception phrase found: |
| 753 |
// Combination exception search term found: |
| 754 |
message_no = searchterms ? 456 : 605; |
| 755 |
whatIsNaughtyLog = o.language_list.getTranslation(message_no); |
| 756 |
whatIsNaughtyLog += combisofar; |
| 757 |
whatIsNaughty = ""; |
| 758 |
++combicurrent; |
| 759 |
cat = *(++combicurrent); |
| 760 |
whatIsNaughtyCategories = o.lm.l[phraselist]->getListCategoryAtD(cat); |
| 761 |
return; |
| 762 |
} else if (type == 1) { // combination weighting |
| 763 |
weight = *(++combicurrent); |
| 764 |
weighting += weight * (foc->weighted_phrase_mode == 2 ? 1 : lowest_occurrences); |
| 765 |
if (weight > 0) { |
| 766 |
cat = *(++combicurrent); |
| 767 |
//category index -1 indicates an uncategorised list |
| 768 |
if (cat >= 0) { |
| 769 |
//don't output duplicate categories |
| 770 |
catcurrent = listcategories.find(cat); |
| 771 |
if (catcurrent != listcategories.end()) { |
| 772 |
catcurrent->second.weight += weight * (foc->weighted_phrase_mode == 2 ? 1 : lowest_occurrences); |
| 773 |
} else { |
| 774 |
currcat = o.lm.l[phraselist]->getListCategoryAtD(cat); |
| 775 |
listcategories[cat] = listent(weight, currcat); |
| 776 |
} |
| 777 |
} |
| 778 |
} else { |
| 779 |
// skip past category for negatively weighted phrases |
| 780 |
combicurrent++; |
| 781 |
} |
| 782 |
if (weightedphrase.length() > 0) { |
| 783 |
weightedphrase += "+"; |
| 784 |
} |
| 785 |
weightedphrase += "("; |
| 786 |
if (weight < 0) { |
| 787 |
weightedphrase += "-" + combisofar; |
| 788 |
} else { |
| 789 |
weightedphrase += combisofar; |
| 790 |
} |
| 791 |
|
| 792 |
DEBUG_content("found combi weighted phrase (", foc->weighted_phrase_mode, "): ", combisofar, |
| 793 |
" x", lowest_occurrences, " (per phrase: ", weight, |
| 794 |
", calculated: ", (weight * (foc->weighted_phrase_mode == 2 ? 1 : lowest_occurrences)), ")" ); |
| 795 |
|
| 796 |
weightedphrase += ")"; |
| 797 |
combisofar = ""; |
| 798 |
} else if (type == 0) { // combination banned |
| 799 |
bannedcombi = true; |
| 800 |
combifound += "(" + combisofar + ")"; |
| 801 |
combisofar = ""; |
| 802 |
combicurrent += 2; |
| 803 |
cat = *(combicurrent); |
| 804 |
bannedcategory = o.lm.l[phraselist]->getListCategoryAtD(cat); |
| 805 |
} |
| 806 |
} else { |
| 807 |
// We had an end marker, but not all the parts so far were matched. |
| 808 |
// Reset the match flag ready for the next chain, and advance to its first part. |
| 809 |
allcmatched = true; |
| 810 |
combicurrent += 4; |
| 811 |
lowest_occurrences = 0; |
| 812 |
} |
| 813 |
} else { |
| 814 |
// We didn't get an end marker - just an individual part. |
| 815 |
// If all parts in the current chain have been matched so far, look for this one as well. |
| 816 |
if (allcmatched) { |
| 817 |
s1 = o.lm.l[phraselist]->getItemAtInt(index); |
| 818 |
if ((foundcurrent = found.find(s1)) == foundend) { |
| 819 |
allcmatched = false; |
| 820 |
combisofar = ""; |
| 821 |
} else { |
| 822 |
if (combisofar.length() > 0) { |
| 823 |
combisofar += ", "; |
| 824 |
} |
| 825 |
combisofar += s1; |
| 826 |
// also track lowest number of times any one part occurs in the text |
| 827 |
// as this will correspond to the number of times the whole chain occurs |
| 828 |
if ((lowest_occurrences == 0) || (lowest_occurrences > foundcurrent->second.second)) { |
| 829 |
lowest_occurrences = foundcurrent->second.second; |
| 830 |
} |
| 831 |
} |
| 832 |
} |
| 833 |
} |
| 834 |
// Advance to the next part in the current chain |
| 835 |
combicurrent++; |
| 836 |
} |
| 837 |
|
| 838 |
// even if we already found a combi ban, we must still wait; there may be non-combi exceptions to follow |
| 839 |
|
| 840 |
// now check non-combi phrases |
| 841 |
foundcurrent = found.begin(); |
| 842 |
while (foundcurrent != foundend) { |
| 843 |
// check time for current phrase |
| 844 |
if (not o.lm.l[phraselist]->checkTimeAt(foundcurrent->second.first)) { |
| 845 |
DEBUG_content("Ignoring phrase based on time limits: ", foundcurrent->first, |
| 846 |
", ", o.lm.l[phraselist]->getListCategoryAt(foundcurrent->second.first) ); |
| 847 |
foundcurrent++; |
| 848 |
continue; |
| 849 |
} |
| 850 |
// 0=banned, 1=weighted, -1=exception, 2=combi, 3=weightedcombi |
| 851 |
type = o.lm.l[phraselist]->getTypeAt(foundcurrent->second.first); |
| 852 |
if (type == 0) { |
| 853 |
// if we already found a combi ban, we don't need to know this stuff |
| 854 |
if (!bannedcombi) { |
| 855 |
isItNaughty = true; |
| 856 |
bannedphrase = foundcurrent->first; |
| 857 |
bannedcategory = o.lm.l[phraselist]->getListCategoryAt(foundcurrent->second.first, &cat); |
| 858 |
} |
| 859 |
} else if (type == 1) { |
| 860 |
// found a weighted phrase - either add one lot of its score, or one lot for every occurrence, depending on phrase filtering mode |
| 861 |
weight = o.lm.l[phraselist]->getWeightAt(foundcurrent->second.first) * (foc->weighted_phrase_mode == 2 ? 1 : foundcurrent->second.second); |
| 862 |
weighting += weight; |
| 863 |
if (weight > 0) { |
| 864 |
currcat = o.lm.l[phraselist]->getListCategoryAt(foundcurrent->second.first, &cat); |
| 865 |
if (cat >= 0) { |
| 866 |
//don't output duplicate categories |
| 867 |
catcurrent = listcategories.find(cat); |
| 868 |
if (catcurrent != listcategories.end()) { |
| 869 |
// add one or N times the weight to this category's score |
| 870 |
catcurrent->second.weight += weight * (foc->weighted_phrase_mode == 2 ? 1 : foundcurrent->second.second); |
| 871 |
} else { |
| 872 |
listcategories[cat] = listent(weight, currcat); |
| 873 |
} |
| 874 |
} |
| 875 |
} |
| 876 |
|
| 877 |
if (o.naughty.show_weighted_found) { |
| 878 |
if (weightedphrase.length() > 0) { |
| 879 |
weightedphrase += "+"; |
| 880 |
} |
| 881 |
if (weight < 0) { |
| 882 |
weightedphrase += "-"; |
| 883 |
} |
| 884 |
|
| 885 |
weightedphrase += foundcurrent->first; |
| 886 |
} |
| 887 |
DEBUG_content("found weighted phrase (", foc->weighted_phrase_mode, "): ", foundcurrent->first, |
| 888 |
" x", foundcurrent->second.second, |
| 889 |
" (per phrase: ", o.lm.l[phraselist]->getWeightAt(foundcurrent->second.first), |
| 890 |
", calculated: ", weight, ")" ); |
| 891 |
} else if (type == -1) { |
| 892 |
isException = true; |
| 893 |
isItNaughty = false; |
| 894 |
// Exception phrase found: |
| 895 |
// Exception search term found: |
| 896 |
message_no = searchterms ? 457 : 604; |
| 897 |
whatIsNaughtyLog = o.language_list.getTranslation(message_no); |
| 898 |
whatIsNaughtyLog += foundcurrent->first; |
| 899 |
whatIsNaughty = ""; |
| 900 |
whatIsNaughtyCategories = o.lm.l[phraselist]->getListCategoryAt(foundcurrent->second.first, NULL); |
| 901 |
return; // no point in going further |
| 902 |
} |
| 903 |
foundcurrent++; |
| 904 |
} |
| 905 |
|
| 906 |
DEBUG_content("WEIGHTING: ", weighting ); |
| 907 |
|
| 908 |
// store the lowest negative weighting or highest positive weighting out of all filtering runs, preferring to store positive weightings. |
| 909 |
if ((weighting < 0 && naughtiness <= 0 && weighting < naughtiness) || (naughtiness >= 0 && weighting > naughtiness) || (naughtiness < 0 && weighting > 0)) { |
| 910 |
naughtiness = weighting; |
| 911 |
} |
| 912 |
|
| 913 |
DEBUG_content("NAUGHTINESS: ", naughtiness ); |
| 914 |
|
| 915 |
// *now* we can safely get down to the whole banning business! |
| 916 |
|
| 917 |
if (bannedcombi) { |
| 918 |
isItNaughty = true; |
| 919 |
// Banned combination phrase found: |
| 920 |
// Banned combination search term found: |
| 921 |
message_no = searchterms ? 453 : 401; |
| 922 |
log_message_no = searchterms ? 452 : 400; |
| 923 |
whatIsNaughtyLog = o.language_list.getTranslation(log_message_no); |
| 924 |
whatIsNaughtyLog += combifound; |
| 925 |
// Banned combination phrase found. |
| 926 |
// Banned combination search term found. |
| 927 |
whatIsNaughty = o.language_list.getTranslation(message_no ); |
| 928 |
whatIsNaughtyCategories = bannedcategory.toCharArray(); |
| 929 |
return; |
| 930 |
} |
| 931 |
|
| 932 |
if (isItNaughty) { |
| 933 |
// Banned phrase found: |
| 934 |
// Banned search term found: |
| 935 |
message_no = searchterms ? 450 : 300; |
| 936 |
whatIsNaughtyLog = o.language_list.getTranslation(message_no); |
| 937 |
whatIsNaughtyLog += bannedphrase; |
| 938 |
// Banned phrase found. |
| 939 |
// Banned search term found. |
| 940 |
whatIsNaughty = o.language_list.getTranslation(searchterms ? 451 : 301); |
| 941 |
whatIsNaughtyCategories = bannedcategory.toCharArray(); |
| 942 |
return; |
| 943 |
} |
| 944 |
|
| 945 |
bool log_it = o.naughty.show_all_weighted_found; |
| 946 |
auto mno = searchterms ? 454 : 402; |
| 947 |
|
| 948 |
if (weighting > limit) { |
| 949 |
log_it = true; |
| 950 |
isItNaughty = true; |
| 951 |
message_no = mno; |
| 952 |
whatIsNaughty = o.language_list.getTranslation(searchterms ? 455 : 403); |
| 953 |
} |
| 954 |
|
| 955 |
if(log_it) { |
| 956 |
// Weighted phrase limit of |
| 957 |
// Weighted search term limit of |
| 958 |
whatIsNaughtyLog = o.language_list.getTranslation(mno); |
| 959 |
whatIsNaughtyLog += String(limit).toCharArray(); |
| 960 |
whatIsNaughtyLog += " : "; |
| 961 |
whatIsNaughtyLog += String(weighting).toCharArray(); |
| 962 |
if (o.naughty.show_weighted_found) { |
| 963 |
whatIsNaughtyLog += " ("; |
| 964 |
whatIsNaughtyLog += weightedphrase; |
| 965 |
whatIsNaughtyLog += ")"; |
| 966 |
} |
| 967 |
// Weighted phrase limit exceeded. |
| 968 |
// Weighted search term limit exceeded. |
| 969 |
// Generate category list, sorted with highest scoring first. |
| 970 |
bool nonempty = false; |
| 971 |
bool belowthreshold = false; |
| 972 |
String categories; |
| 973 |
std::deque<listent> sortable_listcategories; |
| 974 |
catcurrent = listcategories.begin(); |
| 975 |
while (catcurrent != listcategories.end()) { |
| 976 |
sortable_listcategories.push_back(catcurrent->second); |
| 977 |
catcurrent++; |
| 978 |
} |
| 979 |
std::sort(sortable_listcategories.begin(), sortable_listcategories.end()); |
| 980 |
std::deque<listent>::iterator k = sortable_listcategories.begin(); |
| 981 |
while (k != sortable_listcategories.end()) { |
| 982 |
// if category display threshold is in use, apply it |
| 983 |
if (!belowthreshold && (foc->category_threshold > 0) |
| 984 |
&& (k->weight < foc->category_threshold)) { |
| 985 |
whatIsNaughtyDisplayCategories = categories.toCharArray(); |
| 986 |
belowthreshold = true; |
| 987 |
usedisplaycats = true; |
| 988 |
} |
| 989 |
if (k->string.length() > 0) { |
| 990 |
if (nonempty) |
| 991 |
categories += ", "; |
| 992 |
categories += k->string; |
| 993 |
nonempty = true; |
| 994 |
} |
| 995 |
k++; |
| 996 |
// if category threshold is set to show only the top category, |
| 997 |
// everything after the first loop is below the threshold |
| 998 |
if (!belowthreshold && foc->category_threshold < 0) { |
| 999 |
whatIsNaughtyDisplayCategories = categories.toCharArray(); |
| 1000 |
belowthreshold = true; |
| 1001 |
usedisplaycats = true; |
| 1002 |
} |
| 1003 |
} |
| 1004 |
whatIsNaughtyCategories = categories.toCharArray(); |
| 1005 |
} |
| 1006 |
return; |
| 1007 |
// whatIsNaughty is what is displayed in the browser |
| 1008 |
// whatIsNaughtyLog is what is logged in the log file if at all |
| 1009 |
} |
| 1010 |
|
| 1011 |
|
| 1012 |
// strip the URL down to just the IP/hostname, then do an isIPHostname on the result |
| 1013 |
bool NaughtyFilter::isIPHostnameStrip(String url) |
| 1014 |
{ |
| 1015 |
url = url.getHostname(); |
| 1016 |
if(ch_isiphost.match(url.toCharArray(), Rch_isiphost)) |
| 1017 |
return false; |
| 1018 |
else |
| 1019 |
return true; |
| 1020 |
// return ldl->fg[0]->isIPHostname(url); |
| 1021 |
} |
| 1022 |
|
| 1023 |
String NaughtyFilter::getFlags() { |
| 1024 |
String flags = listen_port; |
| 1025 |
flags += ":"; |
| 1026 |
if (!(authrec == nullptr)) { |
| 1027 |
if (authrec->is_transparent) |
| 1028 |
flags += "T"; |
| 1029 |
else if (authrec->is_icap) |
| 1030 |
flags += "I"; |
| 1031 |
else { |
| 1032 |
if(authrec->is_tlsproxy) |
| 1033 |
flags += "L"; |
| 1034 |
else |
| 1035 |
flags += "P"; |
| 1036 |
} |
| 1037 |
|
| 1038 |
if (o.log.addECHtoFlags && hasECH) { |
| 1039 |
flags += "E"; |
| 1040 |
} |
| 1041 |
|
| 1042 |
if (ismitm) |
| 1043 |
flags += "M"; |
| 1044 |
else if (isconnect) |
| 1045 |
flags += "S"; |
| 1046 |
else if (!authrec->is_icap) |
| 1047 |
flags += "H"; |
| 1048 |
|
| 1049 |
flags += ":"; |
| 1050 |
|
| 1051 |
flags += authrec->user_source; |
| 1052 |
flags += ":"; |
| 1053 |
flags += authrec->group_source; |
| 1054 |
} |
| 1055 |
return flags; |
| 1056 |
} |
| 1057 |
|
| 1058 |
String NaughtyFilter::get_lastmatch() { |
| 1059 |
if (anon_url) { |
| 1060 |
return lastmatch.anonimise(); |
| 1061 |
} |
| 1062 |
return lastmatch; |
| 1063 |
} |
| 1064 |
|
| 1065 |
String NaughtyFilter::get_logUrl() { |
| 1066 |
if (anon_url) { |
| 1067 |
String t1 = logurl.before("://"); |
| 1068 |
String t2 = logurl.after("://"); |
| 1069 |
t1 += "://"; |
| 1070 |
t1 += t2.anonimise(); |
| 1071 |
return t1; |
| 1072 |
} |
| 1073 |
return logurl; |
| 1074 |
} |
| 1075 |
|
| 1076 |
String NaughtyFilter::main_category() { |
| 1077 |
String temp = whatIsNaughtyCategories; |
| 1078 |
if (temp.contains(",")) { |
| 1079 |
temp = temp.before(","); |
| 1080 |
} |
| 1081 |
if (temp == "-") temp = ""; |
| 1082 |
temp.toLower(); |
| 1083 |
return temp; |
| 1084 |
} |
| 1085 |
|