| 1 |
// ListContainer - class for both item and phrase lists |
| 2 |
|
| 3 |
// For all support, instructions and copyright go to: |
| 4 |
// http://e2guardian.org/ |
| 5 |
// Released under the GPL v2, with the OpenSSL exception described in the README file. |
| 6 |
|
| 7 |
// INCLUDES |
| 8 |
|
| 9 |
#ifdef HAVE_CONFIG_H |
| 10 |
#include "e2config.h" |
| 11 |
#endif |
| 12 |
|
| 13 |
#include "ListContainer.hpp" |
| 14 |
#include "StoryBoard.hpp" |
| 15 |
#include "OptionContainer.hpp" |
| 16 |
#include "RegExp.hpp" |
| 17 |
#include "Logger.hpp" |
| 18 |
|
| 19 |
#include <algorithm> |
| 20 |
#include <netdb.h> // for gethostby |
| 21 |
#include <cstdlib> |
| 22 |
#include <cstdio> |
| 23 |
#include <ctime> |
| 24 |
#include <unistd.h> |
| 25 |
#include <iostream> |
| 26 |
#include <fstream> |
| 27 |
#include <sys/stat.h> |
| 28 |
#include <sys/time.h> |
| 29 |
#include <list> |
| 30 |
#include <vector> |
| 31 |
|
| 32 |
// GLOBALS |
| 33 |
|
| 34 |
extern OptionContainer o; |
| 35 |
|
| 36 |
// DEFINES |
| 37 |
|
| 38 |
//#define SBDEBUG |
| 39 |
|
| 40 |
// Constructor - set default values |
| 41 |
StoryBoard::StoryBoard() { |
| 42 |
fnt_cnt = 0; |
| 43 |
} |
| 44 |
|
| 45 |
// delete the memory block when the class is destryed |
| 46 |
StoryBoard::~StoryBoard() { |
| 47 |
reset(); |
| 48 |
} |
| 49 |
|
| 50 |
// clear & reset all values |
| 51 |
void StoryBoard::reset() { |
| 52 |
} |
| 53 |
|
| 54 |
|
| 55 |
bool StoryBoard::readFile(const char *filename, ListMeta &LM, bool is_top) { |
| 56 |
if (strlen(filename) < 3) { |
| 57 |
E2LOGGER_error("Storyboard file", filename, " is not defined"); |
| 58 |
return false; |
| 59 |
} |
| 60 |
DEBUG_trace("Reading storyboard file ", filename); |
| 61 |
|
| 62 |
LMeta = &LM; |
| 63 |
std::string linebuffer; // a string line buffer ;) |
| 64 |
String temp; // a String for temporary manipulation |
| 65 |
String line; |
| 66 |
String command; |
| 67 |
String params; |
| 68 |
String action; |
| 69 |
SBFunction curr_function; |
| 70 |
int fnt_id = 0; |
| 71 |
bool overwrite = false; |
| 72 |
bool in_function = false; |
| 73 |
std::ifstream listfile(filename, std::ios::in); // open the file for reading |
| 74 |
if (!listfile.good()) { |
| 75 |
E2LOGGER_error("Error opening Storyboard file (does it exist?): ", filename); |
| 76 |
return false; |
| 77 |
} |
| 78 |
|
| 79 |
String base_dir(filename); |
| 80 |
base_dir.baseDir(); |
| 81 |
|
| 82 |
bool caseinsensitive = true; |
| 83 |
unsigned int line_no = 0; |
| 84 |
while (!listfile.eof()) { // keep going until end of file |
| 85 |
getline(listfile, linebuffer); // grab a line |
| 86 |
++line_no; |
| 87 |
if (linebuffer.length() == 0) { // sanity checking |
| 88 |
continue; |
| 89 |
} |
| 90 |
|
| 91 |
DEBUG_story("Readline ", linebuffer); |
| 92 |
|
| 93 |
line = linebuffer.c_str(); |
| 94 |
line.removeWhiteSpace(); |
| 95 |
|
| 96 |
// handle included list files |
| 97 |
if (line.startsWith(".")) { |
| 98 |
temp = line.after(".Include<").before(">"); |
| 99 |
if (temp.length() > 0) { |
| 100 |
temp.fullPath(base_dir); |
| 101 |
if (!readFile(temp.toCharArray(), *LMeta, false)) { |
| 102 |
listfile.close(); |
| 103 |
return false; |
| 104 |
} else { |
| 105 |
continue; |
| 106 |
} |
| 107 |
} else { |
| 108 |
continue; |
| 109 |
} |
| 110 |
} |
| 111 |
if (line.startsWith("#")) continue; // ignore comment lines |
| 112 |
if (caseinsensitive) |
| 113 |
line.toLower(); |
| 114 |
command = line.before("("); |
| 115 |
command.removeWhiteSpace(); |
| 116 |
params = line.before(")").after("("); |
| 117 |
params.removeWhiteSpace(); |
| 118 |
action = line.after(")"); |
| 119 |
if (action.contains("#")) |
| 120 |
action = action.before("#"); // remove trailing comments |
| 121 |
action.removeWhiteSpace(); |
| 122 |
|
| 123 |
if (command == "function") { |
| 124 |
if (in_function) { // already in another function definition & so assume end of previous function |
| 125 |
curr_function.end(); |
| 126 |
// push function to list |
| 127 |
if (overwrite) |
| 128 |
funct_vec.at(--fnt_id) = curr_function; |
| 129 |
else |
| 130 |
funct_vec.push_back(curr_function); |
| 131 |
} |
| 132 |
in_function = true; |
| 133 |
String temp = params; |
| 134 |
int oldf = 0; |
| 135 |
if ((oldf = getFunctID(temp)) > 0) { |
| 136 |
if (oldf > SB_BI_FUNC_BASE) { // overloadng buildin action |
| 137 |
E2LOGGER_error("SB: error - reserved word used a function name - ", filename, |
| 138 |
" word ", temp); |
| 139 |
return false; |
| 140 |
} else { |
| 141 |
fnt_id = oldf; |
| 142 |
overwrite = true; |
| 143 |
} |
| 144 |
} else { |
| 145 |
fnt_id = ++fnt_cnt; |
| 146 |
overwrite = false; |
| 147 |
} |
| 148 |
curr_function.start(params, fnt_id, line_no, filename); |
| 149 |
continue; |
| 150 |
} |
| 151 |
if (command == "end") { |
| 152 |
if (in_function) { |
| 153 |
curr_function.end(); |
| 154 |
// push function to list |
| 155 |
if (overwrite) |
| 156 |
funct_vec.at(--fnt_id) = curr_function; |
| 157 |
else |
| 158 |
funct_vec.push_back(curr_function); |
| 159 |
in_function = false; |
| 160 |
} // otherwise ignore |
| 161 |
continue; |
| 162 |
} |
| 163 |
if (!curr_function.addline(command, params, action, line_no)) |
| 164 |
return false; |
| 165 |
} |
| 166 |
|
| 167 |
if (in_function) { // at eof so now end |
| 168 |
curr_function.end(); |
| 169 |
// push function to list |
| 170 |
if (overwrite) |
| 171 |
funct_vec.at(--fnt_id) = curr_function; |
| 172 |
else |
| 173 |
funct_vec.push_back(curr_function); |
| 174 |
} |
| 175 |
|
| 176 |
DEBUG_debug("Read storyboard file ", filename, " finished. ", |
| 177 |
" function vect size is ", String(funct_vec.size()), |
| 178 |
" is_top ", String(is_top)); |
| 179 |
|
| 180 |
if (!is_top) return true; |
| 181 |
|
| 182 |
// only do the 2nd pass once all file(s) have been read |
| 183 |
// in top file now do second pass to record action functions ids in command lines and add list_ids |
| 184 |
DEBUG_trace("Read storyboard start 2nd pass checking"); |
| 185 |
|
| 186 |
for (std::vector<SBFunction>::iterator i = funct_vec.begin(); i != funct_vec.end(); i++) { |
| 187 |
for (std::deque<SBFunction::com_rec>::iterator j = i->comm_dq.begin(); j != i->comm_dq.end(); j++) { |
| 188 |
DEBUG_story("Line ", String(j->file_lineno), |
| 189 |
" state is ", String(j->state), |
| 190 |
" actionid ", String(j->action_id), |
| 191 |
" listname ", j->list_name, |
| 192 |
" function ", i->name, |
| 193 |
" id ", String(i->fn_id)); |
| 194 |
|
| 195 |
// check condition |
| 196 |
if (j->state < SB_STATE_TOPIN) { // is an *in condition and requires a list |
| 197 |
std::deque<int> types; |
| 198 |
switch (j->state) { |
| 199 |
case SB_STATE_SITEIN: |
| 200 |
types = {LIST_TYPE_IPSITE, LIST_TYPE_SITE, LIST_TYPE_REGEXP_BOOL}; |
| 201 |
break; |
| 202 |
case SB_STATE_URLIN: |
| 203 |
types = {LIST_TYPE_IPSITE, LIST_TYPE_SITE, |
| 204 |
LIST_TYPE_URL, LIST_TYPE_FILE_EXT, LIST_TYPE_REGEXP_BOOL}; |
| 205 |
break; |
| 206 |
case SB_STATE_SEARCHIN: |
| 207 |
types = {LIST_TYPE_SEARCH}; |
| 208 |
break; |
| 209 |
case SB_STATE_EMBEDDEDIN: |
| 210 |
types = {LIST_TYPE_IPSITE, LIST_TYPE_SITE, LIST_TYPE_URL, LIST_TYPE_REGEXP_BOOL}; |
| 211 |
break; |
| 212 |
case SB_STATE_REFERERIN: |
| 213 |
types = {LIST_TYPE_IPSITE, LIST_TYPE_SITE, LIST_TYPE_URL, LIST_TYPE_REGEXP_BOOL, |
| 214 |
LIST_TYPE_REGEXP_REP}; |
| 215 |
break; |
| 216 |
case SB_STATE_FULLURLIN: |
| 217 |
types = {LIST_TYPE_REGEXP_REP}; |
| 218 |
break; |
| 219 |
case SB_STATE_HEADERIN: |
| 220 |
types = {LIST_TYPE_REGEXP_REP, LIST_TYPE_REGEXP_BOOL}; |
| 221 |
break; |
| 222 |
case SB_STATE_RESHEADERIN: |
| 223 |
types = {LIST_TYPE_REGEXP_REP, LIST_TYPE_REGEXP_BOOL}; |
| 224 |
break; |
| 225 |
case SB_STATE_CLIENTIN: |
| 226 |
types = {LIST_TYPE_IP, LIST_TYPE_SITE, LIST_TYPE_IPMAP}; |
| 227 |
break; |
| 228 |
case SB_STATE_USERIN: |
| 229 |
types = {LIST_TYPE_IPMAP, LIST_TYPE_MAP}; |
| 230 |
break; |
| 231 |
case SB_STATE_EXTENSIONIN: |
| 232 |
types = {LIST_TYPE_FILE_EXT}; |
| 233 |
break; |
| 234 |
case SB_STATE_MIMEIN: |
| 235 |
types = {LIST_TYPE_MIME}; |
| 236 |
break; |
| 237 |
case SB_STATE_USERAGENTIN: |
| 238 |
types = {LIST_TYPE_REGEXP_BOOL}; |
| 239 |
break; |
| 240 |
case SB_STATE_TIMEIN: |
| 241 |
types = {LIST_TYPE_TIME}; |
| 242 |
break; |
| 243 |
case SB_STATE_LISTEN_PORTIN: |
| 244 |
types = {LIST_TYPE_MAP}; |
| 245 |
break; |
| 246 |
case SB_STATE_CATEGORYIN: |
| 247 |
types = {LIST_TYPE_CATEGORY}; |
| 248 |
break; |
| 249 |
} |
| 250 |
bool found = false; |
| 251 |
for (std::deque<int>::iterator k = types.begin(); k != types.end(); k++) { |
| 252 |
ListMeta::list_info *listiptr = LMeta->findListPtr(j->list_name, *k); |
| 253 |
ListMeta::list_info listi; |
| 254 |
if (listiptr) { |
| 255 |
listi = *listiptr; |
| 256 |
DEBUG_story("SB list reference ", String(listi.list_ref), |
| 257 |
" '", listi.name, "' found for ", j->list_name); |
| 258 |
|
| 259 |
if (listi.name.length()) { |
| 260 |
// std::cerr << "used is set " << "" << listi.list_ref << " '" << listi.name << ":" |
| 261 |
// << *k << std::endl; |
| 262 |
listiptr->used = true; |
| 263 |
j->list_id_dq.push_back(listi); |
| 264 |
found = true; |
| 265 |
} |
| 266 |
} |
| 267 |
} |
| 268 |
if (!found) { |
| 269 |
// warning message |
| 270 |
E2LOGGER_error("SB warning: Undefined list ", j->list_name, |
| 271 |
" used at line ", j->file_lineno, |
| 272 |
" of ", i->file_name, " (", filename, ")"); |
| 273 |
} else { |
| 274 |
DEBUG_story("SB ", j->list_name, " matches ", String(j->list_id_dq.size()), " types"); |
| 275 |
} |
| 276 |
|
| 277 |
} |
| 278 |
// check action |
| 279 |
if ((j->action_id = getFunctID(j->action_name)) == 0) { |
| 280 |
// warning message |
| 281 |
E2LOGGER_error("StoryBoard error: Action ", j->action_name, |
| 282 |
" not defined: ", filename, |
| 283 |
" at line ", String(j->file_lineno), |
| 284 |
" of ", i->file_name); |
| 285 |
} |
| 286 |
DEBUG_story("Line ", String(j->file_lineno), |
| 287 |
" state is ", String(j->state), |
| 288 |
" actionid ", String(j->action_id), |
| 289 |
" listname ", j->list_name); |
| 290 |
} |
| 291 |
} |
| 292 |
// check for required functions |
| 293 |
|
| 294 |
|
| 295 |
for (std::vector<ListMeta::list_info>::iterator j = LMeta->list_vec.begin(); j != LMeta->list_vec.end(); j++) { |
| 296 |
if (!j->used) { |
| 297 |
E2LOGGER_error("SB warning: Defined list ", LMeta->list_type(j->type), ":", j->name, |
| 298 |
" is not referenced in the storyboard ", filename); |
| 299 |
} |
| 300 |
} |
| 301 |
|
| 302 |
return true; |
| 303 |
} |
| 304 |
|
| 305 |
unsigned int StoryBoard::getFunctID(String &fname) { |
| 306 |
unsigned int i = 0; |
| 307 |
// check built in functions first |
| 308 |
if (!funct_vec.empty()) { |
| 309 |
i = funct_vec[0].getBIFunctID(fname); |
| 310 |
if (i > 0) return i; |
| 311 |
} |
| 312 |
// check StoryBoard defined functions |
| 313 |
// std::cerr << "Looking for function " << fname << std::endl;; |
| 314 |
for (std::vector<SBFunction>::iterator j = funct_vec.begin(); j != funct_vec.end(); j++) { |
| 315 |
if (j->name == fname) |
| 316 |
return j->fn_id; |
| 317 |
} |
| 318 |
return 0; |
| 319 |
} |
| 320 |
|
| 321 |
|
| 322 |
bool StoryBoard::runFunct(String &fname, NaughtyFilter &cm) { |
| 323 |
return runFunct(getFunctID(fname), cm); |
| 324 |
} |
| 325 |
|
| 326 |
bool StoryBoard::runFunct(unsigned int fID, NaughtyFilter &cm) { |
| 327 |
--fID; |
| 328 |
SBFunction *F = &(funct_vec[fID]); |
| 329 |
bool action_return = false; |
| 330 |
|
| 331 |
E2LOGGER_storytrace("SB:Entering ", F->getName(), |
| 332 |
" line: ", F->file_lineno, " of ", F->file_name); |
| 333 |
|
| 334 |
for (std::deque<SBFunction::com_rec>::iterator i = F->comm_dq.begin(); i != F->comm_dq.end(); i++) { |
| 335 |
bool isListCheck = false; |
| 336 |
bool isMultiListCheck = false; |
| 337 |
bool isHeaderCheck = false; |
| 338 |
HTTPHeader *targetheader = nullptr; |
| 339 |
bool state_result = false; |
| 340 |
ListMeta::list_result res; |
| 341 |
String target; |
| 342 |
String target2; |
| 343 |
String targetful; |
| 344 |
std::deque<url_rec> targetdq; |
| 345 |
|
| 346 |
switch (i->state) { |
| 347 |
case SB_STATE_SITEIN: |
| 348 |
isListCheck = true; |
| 349 |
target = cm.urldomain; |
| 350 |
target2 = target; |
| 351 |
targetful = cm.url; |
| 352 |
if (has_reverse_hosts(targetdq, cm)) |
| 353 |
isMultiListCheck = true; |
| 354 |
break; |
| 355 |
case SB_STATE_URLIN: |
| 356 |
isListCheck = true; |
| 357 |
target = cm.baseurl; |
| 358 |
target2 = cm.urldomain; |
| 359 |
targetful = cm.url; |
| 360 |
if (has_reverse_hosts(targetdq, cm)) |
| 361 |
isMultiListCheck = true; |
| 362 |
break; |
| 363 |
case SB_STATE_FULLURLIN: |
| 364 |
isListCheck = true; |
| 365 |
target = cm.baseurl; |
| 366 |
target2 = cm.urldomain; |
| 367 |
targetful = cm.url; |
| 368 |
if (has_reverse_hosts(targetdq, cm)) |
| 369 |
isMultiListCheck = true; |
| 370 |
break; |
| 371 |
case SB_STATE_SEARCHIN: |
| 372 |
if (cm.isSearch) { |
| 373 |
isListCheck = true; |
| 374 |
target = cm.search_words; |
| 375 |
} |
| 376 |
break; |
| 377 |
case SB_STATE_EMBEDDEDIN: |
| 378 |
if (!cm.deep_urls_checked) { |
| 379 |
cm.deep_urls = deep_urls(cm.baseurl, cm); |
| 380 |
if (cm.deep_urls.size() > 0) |
| 381 |
cm.hasEmbededURL = true; |
| 382 |
} |
| 383 |
if (cm.hasEmbededURL) { |
| 384 |
isMultiListCheck = true; |
| 385 |
targetdq = cm.deep_urls; |
| 386 |
target = "mutli"; |
| 387 |
} |
| 388 |
break; |
| 389 |
case SB_STATE_REFERERIN: |
| 390 |
isListCheck = true; |
| 391 |
target = cm.request_header->getReferer(); // needs spliting before?? |
| 392 |
target2 = target.getHostname(); |
| 393 |
break; |
| 394 |
case SB_STATE_HEADERIN: |
| 395 |
isHeaderCheck = true; |
| 396 |
targetheader = cm.request_header; |
| 397 |
break; |
| 398 |
case SB_STATE_RESHEADERIN: |
| 399 |
isHeaderCheck = true; |
| 400 |
targetheader = cm.response_header; |
| 401 |
break; |
| 402 |
case SB_STATE_CLIENTIN: |
| 403 |
isListCheck = true; |
| 404 |
target = cm.clientip; |
| 405 |
target2 = cm.clienthost; |
| 406 |
break; |
| 407 |
case SB_STATE_USERIN: |
| 408 |
isListCheck = true; |
| 409 |
cm.user.toLower(); |
| 410 |
target = cm.user; |
| 411 |
target2 = cm.clienthost; |
| 412 |
break; |
| 413 |
case SB_STATE_LISTEN_PORTIN: |
| 414 |
isListCheck = true; |
| 415 |
target = cm.listen_port; |
| 416 |
break; |
| 417 |
case SB_STATE_TIMEIN: |
| 418 |
isListCheck = true; |
| 419 |
break; |
| 420 |
case SB_STATE_USERAGENTIN: |
| 421 |
isListCheck = true; |
| 422 |
targetful = cm.request_header->userAgent(); // needs spliting before??target = ""; |
| 423 |
target2 = ""; |
| 424 |
break; |
| 425 |
case SB_STATE_EXTENSIONIN: |
| 426 |
target = cm.response_header->disposition(); |
| 427 |
if (target.length() > 4) |
| 428 |
isListCheck = true; |
| 429 |
target2 = ""; |
| 430 |
break; |
| 431 |
case SB_STATE_MIMEIN: |
| 432 |
target = cm.response_header->getContentType(); |
| 433 |
if (target.length() > 4) |
| 434 |
isListCheck = true; |
| 435 |
target2 = ""; |
| 436 |
break; |
| 437 |
case SB_STATE_CATEGORYIN: |
| 438 |
target = cm.main_category(); |
| 439 |
DEBUG_debug("CAT target is ", target); |
| 440 |
if (target.length() > 2) |
| 441 |
isListCheck = true; |
| 442 |
target2 = ""; |
| 443 |
break; |
| 444 |
case SB_STATE_CONNECT: |
| 445 |
state_result = cm.isconnect; |
| 446 |
break; |
| 447 |
case SB_STATE_GET: |
| 448 |
state_result = (cm.request_header->requestType() == "GET"); |
| 449 |
break; |
| 450 |
case SB_STATE_POST: |
| 451 |
state_result = (cm.request_header->requestType() == "POST"); |
| 452 |
break; |
| 453 |
case SB_STATE_EXCEPTIONSET: |
| 454 |
state_result = cm.isexception; |
| 455 |
break; |
| 456 |
case SB_STATE_SEMIEXCEPTIONSET: |
| 457 |
state_result = cm.issemiexception; |
| 458 |
break; |
| 459 |
case SB_STATE_GREYSET: |
| 460 |
state_result = cm.isGrey; |
| 461 |
break; |
| 462 |
case SB_STATE_BLOCKSET: |
| 463 |
state_result = cm.isBlocked; |
| 464 |
break; |
| 465 |
case SB_STATE_MITMSET: |
| 466 |
state_result = cm.ismitm; |
| 467 |
break; |
| 468 |
case SB_STATE_DONESET: |
| 469 |
state_result = cm.isdone; |
| 470 |
break; |
| 471 |
case SB_STATE_RETURNSET: |
| 472 |
state_result = cm.isReturn; |
| 473 |
break; |
| 474 |
case SB_STATE_REDIRECTSET: |
| 475 |
state_result = cm.urlredirect; |
| 476 |
break; |
| 477 |
case SB_STATE_VIRUSCHECKSET: |
| 478 |
state_result = !cm.noviruscheck; |
| 479 |
break; |
| 480 |
case SB_STATE_BYPASSSET: |
| 481 |
state_result = cm.isbypass; |
| 482 |
break; |
| 483 |
case SB_STATE_BYPASSALLOWEDSET: |
| 484 |
state_result = cm.isbypassallowed; |
| 485 |
break; |
| 486 |
case SB_STATE_INFECTIONBYPASSALLOWEDSET: |
| 487 |
state_result = cm.isinfectionbypassallowed; |
| 488 |
break; |
| 489 |
case SB_STATE_HASSNI: |
| 490 |
state_result = cm.hasSNI; |
| 491 |
break; |
| 492 |
case SB_STATE_TLS: |
| 493 |
state_result = cm.isTLS; |
| 494 |
break; |
| 495 |
case SB_STATE_SITEISIP: |
| 496 |
state_result = cm.isiphost; |
| 497 |
break; |
| 498 |
case SB_STATE_TRUE: |
| 499 |
state_result = true; |
| 500 |
break; |
| 501 |
} |
| 502 |
|
| 503 |
DEBUG_story("SB-Test ", |
| 504 |
" state: ", F->getState(i->state), |
| 505 |
" target: ", target, |
| 506 |
" target2: ", target2, |
| 507 |
" list_check: ", String(isListCheck), |
| 508 |
" targetfull: ", targetful, |
| 509 |
" isSearch: ", String(cm.isSearch), |
| 510 |
" state_result: ", String(state_result)); |
| 511 |
|
| 512 |
if (isHeaderCheck) { |
| 513 |
DEBUG_trace("HeaderCheck"); |
| 514 |
for (std::deque<String>::iterator u = targetheader->header.begin(); |
| 515 |
u != targetheader->header.end(); u++) { |
| 516 |
// String t = *u; |
| 517 |
for (std::deque<ListMeta::list_info>::iterator j = i->list_id_dq.begin(); |
| 518 |
j != i->list_id_dq.end(); j++) { |
| 519 |
DEBUG_story("checking ", j->name, " type ", j->type); |
| 520 |
|
| 521 |
if (LMeta->inList(*j, *u, res)) { //found |
| 522 |
state_result = true; |
| 523 |
if (i->isif) { |
| 524 |
cm.lastcategory = res.category; |
| 525 |
cm.whatIsNaughtyCategories = res.category; |
| 526 |
cm.message_no = res.mess_no; |
| 527 |
cm.log_message_no = res.log_mess_no; |
| 528 |
cm.lastmatch = res.match; |
| 529 |
cm.result = res.result; |
| 530 |
if (j->type == LIST_TYPE_REGEXP_REP) { |
| 531 |
*u = res.result; |
| 532 |
} |
| 533 |
if (o.log.anonymise_logs) |
| 534 |
cm.anon_user = true; |
| 535 |
if (res.anon_log) { |
| 536 |
cm.anon_url = true; |
| 537 |
} |
| 538 |
} |
| 539 |
DEBUG_story("SB lc", cm.lastcategory, " mess_no ", cm.message_no, |
| 540 |
" log_mess ", cm.log_message_no, " match ", res.match); |
| 541 |
break; |
| 542 |
} |
| 543 |
} |
| 544 |
if (state_result) |
| 545 |
break; |
| 546 |
} |
| 547 |
} |
| 548 |
if (isListCheck) { |
| 549 |
DEBUG_story("ListCheck:", i->list_name, " size:", i->list_id_dq.size()); |
| 550 |
for (std::deque<ListMeta::list_info>::iterator j = i->list_id_dq.begin(); j != i->list_id_dq.end(); j++) { |
| 551 |
//ListMeta::list_result res; |
| 552 |
String t; |
| 553 |
if ((j->type >= LIST_TYPE_SITE) && (j->type < LIST_TYPE_URL)) { |
| 554 |
t = target2; |
| 555 |
} else if (j->type == LIST_TYPE_REGEXP_BOOL || j->type == LIST_TYPE_REGEXP_REP) { |
| 556 |
t = targetful; |
| 557 |
} else if ((j->type == LIST_TYPE_FILE_EXT) && target.contains("?")) { |
| 558 |
t = target.before("?"); // remove cgi part of url |
| 559 |
} else { |
| 560 |
t = target; |
| 561 |
} |
| 562 |
|
| 563 |
DEBUG_story("ListCheck ", j->name, "(type ", j->type, ")", " for ", t); |
| 564 |
if (cm.issiteonly && (j->type == LIST_TYPE_URL || j->type == LIST_TYPE_FILE_EXT)) |
| 565 |
continue; |
| 566 |
if (!(cm.isiphost) && j->type == LIST_TYPE_IPSITE) |
| 567 |
continue; |
| 568 |
if ((cm.isiphost) && j->type == LIST_TYPE_SITE && !o.lists.search_sitelist_for_ip) |
| 569 |
continue; |
| 570 |
|
| 571 |
// DEBUG_story("checking ", j->name, " type ", String(j->type)); |
| 572 |
|
| 573 |
if (LMeta->inList(*j, t, res)) { //found |
| 574 |
state_result = true; |
| 575 |
if (i->isif) { |
| 576 |
cm.result = res.result; |
| 577 |
} |
| 578 |
|
| 579 |
DEBUG_story("ListCheck lc", cm.lastcategory, " mess_no ", cm.message_no, |
| 580 |
" log_mess ", cm.log_message_no, " match ", res.match); |
| 581 |
break; |
| 582 |
} |
| 583 |
} |
| 584 |
DEBUG_debug("ListCheck:", i->list_name, " NOT FOUND"); |
| 585 |
} |
| 586 |
if (isMultiListCheck && !state_result) { |
| 587 |
DEBUG_trace("MultiListCheck"); |
| 588 |
for (std::deque<url_rec>::iterator u = targetdq.begin(); u != targetdq.end(); u++) { |
| 589 |
|
| 590 |
for (std::deque<ListMeta::list_info>::iterator j = i->list_id_dq.begin(); |
| 591 |
j != i->list_id_dq.end(); j++) { |
| 592 |
//ListMeta::list_result res; |
| 593 |
String t; |
| 594 |
if ((j->type >= LIST_TYPE_SITE) && (j->type < LIST_TYPE_URL)) { |
| 595 |
t = u->urldomain; |
| 596 |
} else if (j->type == LIST_TYPE_REGEXP_BOOL || j->type == LIST_TYPE_REGEXP_REP) { |
| 597 |
t = u->fullurl; |
| 598 |
} else { |
| 599 |
t = u->baseurl; |
| 600 |
} |
| 601 |
if (u->is_siteonly && j->type == LIST_TYPE_URL) |
| 602 |
continue; |
| 603 |
if (!(u->site_is_ip) && j->type == LIST_TYPE_IPSITE) |
| 604 |
continue; |
| 605 |
if ((u->site_is_ip) && j->type == LIST_TYPE_SITE && !o.lists.search_sitelist_for_ip) |
| 606 |
continue; |
| 607 |
|
| 608 |
DEBUG_story("checking ", j->name, " type ", j->type, "Target ", t); |
| 609 |
if (LMeta->inList(*j, t, res)) { //found |
| 610 |
state_result = true; |
| 611 |
if (i->isif) { |
| 612 |
cm.result = res.result; |
| 613 |
} |
| 614 |
|
| 615 |
DEBUG_story("SB lc", cm.lastcategory, " mess_no ", cm.message_no, " log_mess ", |
| 616 |
cm.log_message_no, " match ", res.match); |
| 617 |
break; |
| 618 |
} |
| 619 |
} |
| 620 |
if (state_result) break; |
| 621 |
} |
| 622 |
} |
| 623 |
if (!i->isif) { |
| 624 |
state_result = !state_result; |
| 625 |
} |
| 626 |
DEBUG_story("SB-Result", |
| 627 |
" state: ", F->getState(i->state), |
| 628 |
" target: ", target, |
| 629 |
" target2: ", target2, |
| 630 |
" list_check: ", String(isListCheck), |
| 631 |
" targetfull: ", targetful, |
| 632 |
" isSearch: ", String(cm.isSearch), |
| 633 |
" state_result: ", String(state_result)); |
| 634 |
|
| 635 |
E2LOGGER_storytrace("SB: ", i->file_lineno, |
| 636 |
(i->isif ? " if(" : " ifnot("), |
| 637 |
F->getState(i->state), ",", |
| 638 |
i->list_name, ") is ", |
| 639 |
(state_result ? "true" : "false")); |
| 640 |
|
| 641 |
if (!state_result) { |
| 642 |
action_return = false; |
| 643 |
cm.isReturn = action_return; |
| 644 |
continue; // nothing to do so continue to next SB line |
| 645 |
} |
| 646 |
|
| 647 |
|
| 648 |
action_return = true; |
| 649 |
|
| 650 |
// if (i->mess_no > 0) cm.message_no = i->mess_no; |
| 651 |
// if (i->log_mess_no > 0) cm.log_message_no = i->log_mess_no; |
| 652 |
|
| 653 |
DEBUG_story("ACTION ", |
| 654 |
" lc: ", cm.lastcategory, |
| 655 |
" mess_no: ", String(cm.message_no), |
| 656 |
" log_mess: ", String(cm.log_message_no), |
| 657 |
" match: ", cm.whatIsNaughty, |
| 658 |
" action_id: ", F->getBIFunct(i->action_id)); |
| 659 |
|
| 660 |
if (i->action_id > SB_BI_FUNC_BASE) { // is built-in action |
| 661 |
switch (i->action_id) { |
| 662 |
case SB_FUNC_SETEXCEPTION: |
| 663 |
cm.isexception = true; |
| 664 |
cm.isGrey = false; |
| 665 |
cm.isBlocked = false; |
| 666 |
if (!cm.issemiexception) { // don't overwrite semi exception messages |
| 667 |
update_messages(cm, res); |
| 668 |
if (i->mess_no > 0) cm.message_no = i->mess_no; |
| 669 |
if (i->log_mess_no > 0) cm.log_message_no = i->log_mess_no; |
| 670 |
cm.whatIsNaughty = o.language_list.getTranslation(cm.message_no) + cm.lastmatch; |
| 671 |
if (cm.log_message_no == 0) |
| 672 |
cm.whatIsNaughtyLog = cm.whatIsNaughty; |
| 673 |
else |
| 674 |
cm.whatIsNaughtyLog = o.language_list.getTranslation(cm.log_message_no) + cm.lastmatch; |
| 675 |
cm.exceptioncat = cm.lastcategory; |
| 676 |
} |
| 677 |
cm.issemiexception = false; |
| 678 |
break; |
| 679 |
case SB_FUNC_SETSEMIEXCEPTION: |
| 680 |
cm.issemiexception = true; |
| 681 |
cm.isexception = false; |
| 682 |
cm.isGrey = false; |
| 683 |
cm.isBlocked = false; |
| 684 |
update_messages(cm, res); |
| 685 |
if (i->mess_no > 0) cm.message_no = i->mess_no; |
| 686 |
if (i->log_mess_no > 0) cm.log_message_no = i->log_mess_no; |
| 687 |
//cm.exceptionreason = o.language_list.getTranslation(cm.message_no); |
| 688 |
cm.whatIsNaughty = o.language_list.getTranslation(cm.message_no) + cm.lastmatch; |
| 689 |
if (cm.log_message_no == 0) |
| 690 |
cm.whatIsNaughtyLog = cm.whatIsNaughty; |
| 691 |
else |
| 692 |
cm.whatIsNaughtyLog = o.language_list.getTranslation(cm.log_message_no) + cm.lastmatch; |
| 693 |
cm.exceptioncat = cm.lastcategory; |
| 694 |
break; |
| 695 |
case SB_FUNC_UNSETSEMIEXCEPTION: |
| 696 |
cm.issemiexception = false; |
| 697 |
break; |
| 698 |
case SB_FUNC_SETGREY: |
| 699 |
cm.isGrey = true; |
| 700 |
cm.isexception = false; |
| 701 |
cm.issemiexception = false; |
| 702 |
cm.isBlocked = false; |
| 703 |
break; |
| 704 |
case SB_FUNC_SETBLOCK: |
| 705 |
cm.isBlocked = true; |
| 706 |
cm.isGrey = false; |
| 707 |
cm.isexception = false; |
| 708 |
cm.issemiexception = false; |
| 709 |
update_messages(cm, res); |
| 710 |
if (i->mess_no > 0) cm.message_no = i->mess_no; |
| 711 |
if (i->log_mess_no > 0) cm.log_message_no = i->log_mess_no; |
| 712 |
if (cm.message_no == 503) |
| 713 |
cm.whatIsNaughty = o.language_list.getTranslation(cm.message_no); |
| 714 |
else |
| 715 |
cm.whatIsNaughty = o.language_list.getTranslation(cm.message_no) + cm.get_lastmatch(); |
| 716 |
if (cm.log_message_no == 0) |
| 717 |
cm.whatIsNaughtyLog = o.language_list.getTranslation(cm.message_no) + cm.get_lastmatch(); |
| 718 |
else |
| 719 |
cm.whatIsNaughtyLog = o.language_list.getTranslation(cm.log_message_no) + cm.get_lastmatch(); |
| 720 |
cm.whatIsNaughtyCategories = cm.lastcategory; |
| 721 |
break; |
| 722 |
case SB_FUNC_SETMODURL: |
| 723 |
cm.urlmodified = true; |
| 724 |
cm.request_header->setURL(cm.result); |
| 725 |
cm.url = cm.result; |
| 726 |
cm.baseurl = cm.url; |
| 727 |
cm.baseurl.removeWhiteSpace(); |
| 728 |
cm.baseurl.toLower(); |
| 729 |
cm.baseurl.removePTP(); |
| 730 |
cm.logurl = cm.request_header->getLogUrl(false, cm.ismitm); |
| 731 |
cm.urld = cm.request_header->decode(cm.url); |
| 732 |
cm.urldomain = cm.url.getHostname(); |
| 733 |
cm.urldomain.toLower(); |
| 734 |
cm.connect_site = cm.urldomain; |
| 735 |
DEBUG_story("SB: URL modified to ", cm.url); |
| 736 |
break; |
| 737 |
case SB_FUNC_SETCONNECTSITE: |
| 738 |
cm.urlmodified = true; |
| 739 |
cm.logurl = cm.result; |
| 740 |
cm.connect_site = cm.result.getHostname(); |
| 741 |
DEBUG_story("SB: connect site changed to ", cm.connect_site); |
| 742 |
break; |
| 743 |
case SB_FUNC_SETLOGCAT: |
| 744 |
cm.logcategory = true; |
| 745 |
update_messages(cm, res); |
| 746 |
if (i->mess_no > 0) cm.message_no = i->mess_no; |
| 747 |
if (i->log_mess_no > 0) cm.log_message_no = i->log_mess_no; |
| 748 |
cm.whatIsNaughty = o.language_list.getTranslation(cm.message_no) + cm.lastmatch; |
| 749 |
if (cm.log_message_no == 0) |
| 750 |
cm.whatIsNaughtyLog = cm.whatIsNaughty; |
| 751 |
else |
| 752 |
cm.whatIsNaughtyLog = o.language_list.getTranslation(cm.log_message_no) + cm.lastmatch; |
| 753 |
cm.whatIsNaughtyCategories = cm.lastcategory; |
| 754 |
break; |
| 755 |
case SB_FUNC_SETREDIRECT: |
| 756 |
if (cm.result.size() > 0) { |
| 757 |
cm.request_header->redirect = cm.result; |
| 758 |
cm.urlredirect = true; |
| 759 |
} else { |
| 760 |
action_return = false; |
| 761 |
} |
| 762 |
break; |
| 763 |
case SB_FUNC_SETGOMITM: |
| 764 |
if (cm.ismitmcandidate && !cm.nomitm) { |
| 765 |
cm.gomitm = true; |
| 766 |
} else { |
| 767 |
action_return = false; |
| 768 |
} |
| 769 |
break; |
| 770 |
case SB_FUNC_SETNOMITM: |
| 771 |
cm.nomitm = true; |
| 772 |
cm.gomitm = false; |
| 773 |
cm.automitm = false; |
| 774 |
break; |
| 775 |
case SB_FUNC_SETAUTOMITM: |
| 776 |
if (!cm.nomitm) cm.automitm = true; |
| 777 |
break; |
| 778 |
case SB_FUNC_UNSETAUTOMITM: |
| 779 |
cm.automitm = false; |
| 780 |
break; |
| 781 |
case SB_FUNC_SETADDHEADER: |
| 782 |
cm.headeradded = true; |
| 783 |
cm.request_header->addHeader(cm.result); |
| 784 |
break; |
| 785 |
case SB_FUNC_SETMODHEADER: |
| 786 |
cm.headermodified = true; |
| 787 |
break; |
| 788 |
case SB_FUNC_SETNOCHECKCERT: |
| 789 |
cm.nocheckcert = true; |
| 790 |
break; |
| 791 |
case SB_FUNC_SETSEARCHTERM: |
| 792 |
if (cm.result.size() > 0) { |
| 793 |
cm.isSearch = true; |
| 794 |
cm.search_words = cm.result.sort_search(); |
| 795 |
cm.search_terms = cm.result; |
| 796 |
cm.search_terms.swapChar('+', ' '); |
| 797 |
}; |
| 798 |
break; |
| 799 |
case SB_FUNC_SETGODIRECT: |
| 800 |
cm.isdirect = true; |
| 801 |
break; |
| 802 |
case SB_FUNC_SETDONE: |
| 803 |
cm.isdone = true; |
| 804 |
break; |
| 805 |
case SB_FUNC_SETNOLOG: |
| 806 |
cm.nolog = true; |
| 807 |
break; |
| 808 |
case SB_FUNC_SETALERT: |
| 809 |
cm.alert = true; |
| 810 |
break; |
| 811 |
case SB_FUNC_SETGROUP: |
| 812 |
action_return = false; |
| 813 |
if (cm.result.size() > 0) { |
| 814 |
int g = cm.result.toInteger(); |
| 815 |
if (g > 0 && g <= o.filter.numfg) { |
| 816 |
cm.filtergroup = --g; |
| 817 |
cm.authrec->group_source = i->list_name; |
| 818 |
action_return = true; |
| 819 |
} |
| 820 |
}; |
| 821 |
break; |
| 822 |
case SB_FUNC_UNSETVIRUSCHECK: |
| 823 |
cm.noviruscheck = true; |
| 824 |
break; |
| 825 |
case SB_FUNC_UNSETBYPASS: |
| 826 |
cm.isbypass = false; |
| 827 |
cm.iscookiebypass = false; |
| 828 |
cm.isscanbypass = false; |
| 829 |
cm.isvirusbypass = false; |
| 830 |
cm.isexception = false; |
| 831 |
break; |
| 832 |
case SB_FUNC_UNSETBYPASSALLOW: |
| 833 |
if (cm.isbypassallowed) |
| 834 |
cm.isbypassallowed = false; |
| 835 |
else |
| 836 |
action_return = false; |
| 837 |
break; |
| 838 |
case SB_FUNC_UNSETINFECTIONBYPASSALLOW: |
| 839 |
if (cm.isinfectionbypassallowed) |
| 840 |
cm.isinfectionbypassallowed = false; |
| 841 |
else |
| 842 |
action_return = false; |
| 843 |
break; |
| 844 |
case SB_FUNC_SETTRUE: |
| 845 |
break; |
| 846 |
case SB_FUNC_SETFALSE: |
| 847 |
action_return = false; |
| 848 |
break; |
| 849 |
} |
| 850 |
// DEBUG_story("SB-ACTION: ", F->name, " : ", F->getBIFunct(i->action_id), " ", (action_return? "true" : "false" )); |
| 851 |
} else { // is SB defined function |
| 852 |
|
| 853 |
if (i->action_id > 0) { |
| 854 |
action_return = runFunct(i->action_id, cm); |
| 855 |
E2LOGGER_storytrace("SB:Resuming ", F->name, " in ", F->file_name); |
| 856 |
} |
| 857 |
|
| 858 |
} |
| 859 |
cm.isReturn = action_return; |
| 860 |
if (i->return_after_action) |
| 861 |
break; |
| 862 |
if (i->return_after_action_is_true && action_return) |
| 863 |
break; |
| 864 |
} |
| 865 |
|
| 866 |
E2LOGGER_storytrace("SB:Exiting ", F->getName(), " returned ", (action_return ? "true" : "false")); |
| 867 |
|
| 868 |
return action_return; |
| 869 |
} |
| 870 |
|
| 871 |
bool StoryBoard::setEntry(unsigned int index, String fname) { |
| 872 |
entrys[index] = getFunctID(fname); |
| 873 |
if (entrys[index] > 0) { |
| 874 |
return true; |
| 875 |
} |
| 876 |
return false; |
| 877 |
}; |
| 878 |
|
| 879 |
bool StoryBoard::runFunctEntry(unsigned int index, NaughtyFilter &cm) { |
| 880 |
DEBUG_trace(""); |
| 881 |
cm.isdone = false; // only has logical scope for a single call |
| 882 |
if (entrys[index] > 0) |
| 883 |
return runFunct(entrys[index], cm); |
| 884 |
else |
| 885 |
return false; |
| 886 |
}; |
| 887 |
|
| 888 |
std::deque<url_rec> StoryBoard::deep_urls(String &urld, NaughtyFilter &cm) { |
| 889 |
std::deque<url_rec> temp; |
| 890 |
String durl = urld; |
| 891 |
|
| 892 |
// Google web site translate checking |
| 893 |
// Format is domain with '.' replaced with '-' and '-' with '--' ending with .translate.goo |
| 894 |
// so www.merriam-webster.com -> www-merriam--webster-com.translate.goog |
| 895 |
|
| 896 |
if (cm.urldomain.endsWith(".translate.goog")) { |
| 897 |
// E2LOGGER_info("Found translate.goog ", durl ); |
| 898 |
String tem_dom = cm.urldomain.before("."); |
| 899 |
String new_url; |
| 900 |
|
| 901 |
while (tem_dom.contains("-")) { |
| 902 |
new_url += tem_dom.before("-"); |
| 903 |
tem_dom = tem_dom.after("-"); |
| 904 |
if (tem_dom.startsWith("-")) { // is a '--' so replace with '-' |
| 905 |
new_url += "-"; |
| 906 |
tem_dom = tem_dom.after("-"); |
| 907 |
} else { |
| 908 |
new_url += "."; // is a '-' so replace with '.' |
| 909 |
} |
| 910 |
// E2LOGGER_info(" GT embedd url is ", new_url); |
| 911 |
} |
| 912 |
new_url += tem_dom; |
| 913 |
// E2LOGGER_info(" GT embedd url is ", new_url); |
| 914 |
|
| 915 |
if (!cm.issiteonly) { //replace url part if needed |
| 916 |
new_url += "/"; |
| 917 |
new_url += urld.after("/"); |
| 918 |
} |
| 919 |
// E2LOGGER_info(" GT embedd url is ", new_url); |
| 920 |
// E2LOGGER_info(" GT tem_dom is ", tem_dom); |
| 921 |
url_rec t; |
| 922 |
t.baseurl = new_url; |
| 923 |
t.fullurl = new_url; |
| 924 |
t.urldomain = t.baseurl.getHostname(); |
| 925 |
if (t.baseurl == t.urldomain) |
| 926 |
t.is_siteonly = true; |
| 927 |
if (cm.isIPHostnameStrip(t.urldomain)) |
| 928 |
t.site_is_ip = true; |
| 929 |
temp.push_back(t); |
| 930 |
} |
| 931 |
|
| 932 |
while (durl.contains(":")) { |
| 933 |
durl = durl.after(":"); |
| 934 |
if (!durl.contains(".")) |
| 935 |
break; |
| 936 |
while (durl.startsWith(":'") || durl.startsWith("/")) { |
| 937 |
durl.lop(); |
| 938 |
} |
| 939 |
if (durl.size() > 5) { |
| 940 |
url_rec t; |
| 941 |
t.baseurl = durl; |
| 942 |
t.baseurl.removePTP(); |
| 943 |
t.fullurl = durl; |
| 944 |
if (durl.startsWith("http:") || durl.startsWith("https:")) |
| 945 |
durl = durl.after(":"); |
| 946 |
t.urldomain = t.baseurl.getHostname(); |
| 947 |
if (t.baseurl == t.urldomain) |
| 948 |
t.is_siteonly = true; |
| 949 |
if (cm.isIPHostnameStrip(t.urldomain)) |
| 950 |
t.site_is_ip = true; |
| 951 |
temp.push_back(t); |
| 952 |
} else { |
| 953 |
break; |
| 954 |
} |
| 955 |
} |
| 956 |
return temp; |
| 957 |
} |
| 958 |
|
| 959 |
// reverse DNS lookup on IP. be aware that this can return multiple results, unlike a standard lookup. |
| 960 |
std::deque<url_rec> StoryBoard::ipToHostname(NaughtyFilter &cm) { |
| 961 |
std::deque<url_rec> result; |
| 962 |
const char *ip = cm.urldomain.c_str(); |
| 963 |
String urlp = cm.urld.after("/"); |
| 964 |
//struct in_addr address, **addrptr; |
| 965 |
struct in_addr address; |
| 966 |
if (inet_aton(ip, &address)) { // convert to in_addr |
| 967 |
struct hostent *answer; |
| 968 |
answer = gethostbyaddr((char *) &address, sizeof(address), AF_INET); |
| 969 |
if (answer) { // sucess in reverse dns |
| 970 |
url_rec t; |
| 971 |
t.urldomain = answer->h_name; |
| 972 |
t.baseurl = t.urldomain + "/" + urlp; |
| 973 |
t.fullurl = "http://" + t.baseurl; |
| 974 |
result.push_back(t); |
| 975 |
//for (addrptr = (struct in_addr **)answer->h_addr_list; *addrptr; addrptr++) { |
| 976 |
// result->push_back(String(inet_ntoa(**addrptr))); |
| 977 |
//} |
| 978 |
} |
| 979 |
} |
| 980 |
return result; |
| 981 |
} |
| 982 |
|
| 983 |
bool StoryBoard::has_reverse_hosts(std::deque<url_rec> &urec, NaughtyFilter &cm) { |
| 984 |
if (!(cm.isiphost && o.story.reverse_lookups)) |
| 985 |
return false; |
| 986 |
if (!cm.reverse_checked) { |
| 987 |
cm.reversedURLs = ipToHostname(cm); |
| 988 |
cm.reverse_checked = true; |
| 989 |
} |
| 990 |
if (cm.reversedURLs.size() > 0) { |
| 991 |
urec = cm.reversedURLs; |
| 992 |
return true; |
| 993 |
} |
| 994 |
return false; |
| 995 |
} |
| 996 |
|
| 997 |
void StoryBoard::update_messages(NaughtyFilter &cm, ListMeta::list_result &res) { |
| 998 |
cm.lastcategory = res.category; |
| 999 |
cm.whatIsNaughtyCategories = res.category; |
| 1000 |
cm.message_no = res.mess_no; |
| 1001 |
cm.log_message_no = res.log_mess_no; |
| 1002 |
cm.lastmatch = res.match; |
| 1003 |
cm.result = res.result; |
| 1004 |
if (res.anon_log) { |
| 1005 |
cm.anon_user = true; |
| 1006 |
cm.anon_url = true; |
| 1007 |
} |
| 1008 |
} |