// ListContainer - class for both item and phrase lists // For all support, instructions and copyright go to: // http://e2guardian.org/ // Released under the GPL v2, with the OpenSSL exception described in the README file. // INCLUDES #ifdef HAVE_CONFIG_H #include "e2config.h" #endif #include "ListContainer.hpp" #include "StoryBoard.hpp" #include "OptionContainer.hpp" #include "RegExp.hpp" #include "Logger.hpp" #include #include // for gethostby #include #include #include #include #include #include #include #include #include #include // GLOBALS extern OptionContainer o; // DEFINES //#define SBDEBUG // Constructor - set default values StoryBoard::StoryBoard() { fnt_cnt = 0; } // delete the memory block when the class is destryed StoryBoard::~StoryBoard() { reset(); } // clear & reset all values void StoryBoard::reset() { } bool StoryBoard::readFile(const char *filename, ListMeta &LM, bool is_top) { if (strlen(filename) < 3) { E2LOGGER_error("Storyboard file", filename, " is not defined"); return false; } DEBUG_trace("Reading storyboard file ", filename); LMeta = &LM; std::string linebuffer; // a string line buffer ;) String temp; // a String for temporary manipulation String line; String command; String params; String action; SBFunction curr_function; int fnt_id = 0; bool overwrite = false; bool in_function = false; std::ifstream listfile(filename, std::ios::in); // open the file for reading if (!listfile.good()) { E2LOGGER_error("Error opening Storyboard file (does it exist?): ", filename); return false; } String base_dir(filename); base_dir.baseDir(); bool caseinsensitive = true; unsigned int line_no = 0; while (!listfile.eof()) { // keep going until end of file getline(listfile, linebuffer); // grab a line ++line_no; if (linebuffer.length() == 0) { // sanity checking continue; } DEBUG_story("Readline ", linebuffer); line = linebuffer.c_str(); line.removeWhiteSpace(); // handle included list files if (line.startsWith(".")) { temp = line.after(".Include<").before(">"); if (temp.length() > 0) { temp.fullPath(base_dir); if (!readFile(temp.toCharArray(), *LMeta, false)) { listfile.close(); return false; } else { continue; } } else { continue; } } if (line.startsWith("#")) continue; // ignore comment lines if (caseinsensitive) line.toLower(); command = line.before("("); command.removeWhiteSpace(); params = line.before(")").after("("); params.removeWhiteSpace(); action = line.after(")"); if (action.contains("#")) action = action.before("#"); // remove trailing comments action.removeWhiteSpace(); if (command == "function") { if (in_function) { // already in another function definition & so assume end of previous function curr_function.end(); // push function to list if (overwrite) funct_vec.at(--fnt_id) = curr_function; else funct_vec.push_back(curr_function); } in_function = true; String temp = params; int oldf = 0; if ((oldf = getFunctID(temp)) > 0) { if (oldf > SB_BI_FUNC_BASE) { // overloadng buildin action E2LOGGER_error("SB: error - reserved word used a function name - ", filename, " word ", temp); return false; } else { fnt_id = oldf; overwrite = true; } } else { fnt_id = ++fnt_cnt; overwrite = false; } curr_function.start(params, fnt_id, line_no, filename); continue; } if (command == "end") { if (in_function) { curr_function.end(); // push function to list if (overwrite) funct_vec.at(--fnt_id) = curr_function; else funct_vec.push_back(curr_function); in_function = false; } // otherwise ignore continue; } if (!curr_function.addline(command, params, action, line_no)) return false; } if (in_function) { // at eof so now end curr_function.end(); // push function to list if (overwrite) funct_vec.at(--fnt_id) = curr_function; else funct_vec.push_back(curr_function); } DEBUG_debug("Read storyboard file ", filename, " finished. ", " function vect size is ", String(funct_vec.size()), " is_top ", String(is_top)); if (!is_top) return true; // only do the 2nd pass once all file(s) have been read // in top file now do second pass to record action functions ids in command lines and add list_ids DEBUG_trace("Read storyboard start 2nd pass checking"); for (std::vector::iterator i = funct_vec.begin(); i != funct_vec.end(); i++) { for (std::deque::iterator j = i->comm_dq.begin(); j != i->comm_dq.end(); j++) { DEBUG_story("Line ", String(j->file_lineno), " state is ", String(j->state), " actionid ", String(j->action_id), " listname ", j->list_name, " function ", i->name, " id ", String(i->fn_id)); // check condition if (j->state < SB_STATE_TOPIN) { // is an *in condition and requires a list std::deque types; switch (j->state) { case SB_STATE_SITEIN: types = {LIST_TYPE_IPSITE, LIST_TYPE_SITE, LIST_TYPE_REGEXP_BOOL}; break; case SB_STATE_URLIN: types = {LIST_TYPE_IPSITE, LIST_TYPE_SITE, LIST_TYPE_URL, LIST_TYPE_FILE_EXT, LIST_TYPE_REGEXP_BOOL}; break; case SB_STATE_SEARCHIN: types = {LIST_TYPE_SEARCH}; break; case SB_STATE_EMBEDDEDIN: types = {LIST_TYPE_IPSITE, LIST_TYPE_SITE, LIST_TYPE_URL, LIST_TYPE_REGEXP_BOOL}; break; case SB_STATE_REFERERIN: types = {LIST_TYPE_IPSITE, LIST_TYPE_SITE, LIST_TYPE_URL, LIST_TYPE_REGEXP_BOOL, LIST_TYPE_REGEXP_REP}; break; case SB_STATE_FULLURLIN: types = {LIST_TYPE_REGEXP_REP}; break; case SB_STATE_HEADERIN: types = {LIST_TYPE_REGEXP_REP, LIST_TYPE_REGEXP_BOOL}; break; case SB_STATE_RESHEADERIN: types = {LIST_TYPE_REGEXP_REP, LIST_TYPE_REGEXP_BOOL}; break; case SB_STATE_CLIENTIN: types = {LIST_TYPE_IP, LIST_TYPE_SITE, LIST_TYPE_IPMAP}; break; case SB_STATE_USERIN: types = {LIST_TYPE_IPMAP, LIST_TYPE_MAP}; break; case SB_STATE_EXTENSIONIN: types = {LIST_TYPE_FILE_EXT}; break; case SB_STATE_MIMEIN: types = {LIST_TYPE_MIME}; break; case SB_STATE_USERAGENTIN: types = {LIST_TYPE_REGEXP_BOOL}; break; case SB_STATE_TIMEIN: types = {LIST_TYPE_TIME}; break; case SB_STATE_LISTEN_PORTIN: types = {LIST_TYPE_MAP}; break; case SB_STATE_CATEGORYIN: types = {LIST_TYPE_CATEGORY}; break; } bool found = false; for (std::deque::iterator k = types.begin(); k != types.end(); k++) { ListMeta::list_info *listiptr = LMeta->findListPtr(j->list_name, *k); ListMeta::list_info listi; if (listiptr) { listi = *listiptr; DEBUG_story("SB list reference ", String(listi.list_ref), " '", listi.name, "' found for ", j->list_name); if (listi.name.length()) { // std::cerr << "used is set " << "" << listi.list_ref << " '" << listi.name << ":" // << *k << std::endl; listiptr->used = true; j->list_id_dq.push_back(listi); found = true; } } } if (!found) { // warning message E2LOGGER_error("SB warning: Undefined list ", j->list_name, " used at line ", j->file_lineno, " of ", i->file_name, " (", filename, ")"); } else { DEBUG_story("SB ", j->list_name, " matches ", String(j->list_id_dq.size()), " types"); } } // check action if ((j->action_id = getFunctID(j->action_name)) == 0) { // warning message E2LOGGER_error("StoryBoard error: Action ", j->action_name, " not defined: ", filename, " at line ", String(j->file_lineno), " of ", i->file_name); } DEBUG_story("Line ", String(j->file_lineno), " state is ", String(j->state), " actionid ", String(j->action_id), " listname ", j->list_name); } } // check for required functions for (std::vector::iterator j = LMeta->list_vec.begin(); j != LMeta->list_vec.end(); j++) { if (!j->used) { E2LOGGER_error("SB warning: Defined list ", LMeta->list_type(j->type), ":", j->name, " is not referenced in the storyboard ", filename); } } return true; } unsigned int StoryBoard::getFunctID(String &fname) { unsigned int i = 0; // check built in functions first if (!funct_vec.empty()) { i = funct_vec[0].getBIFunctID(fname); if (i > 0) return i; } // check StoryBoard defined functions // std::cerr << "Looking for function " << fname << std::endl;; for (std::vector::iterator j = funct_vec.begin(); j != funct_vec.end(); j++) { if (j->name == fname) return j->fn_id; } return 0; } bool StoryBoard::runFunct(String &fname, NaughtyFilter &cm) { return runFunct(getFunctID(fname), cm); } bool StoryBoard::runFunct(unsigned int fID, NaughtyFilter &cm) { --fID; SBFunction *F = &(funct_vec[fID]); bool action_return = false; E2LOGGER_storytrace("SB:Entering ", F->getName(), " line: ", F->file_lineno, " of ", F->file_name); for (std::deque::iterator i = F->comm_dq.begin(); i != F->comm_dq.end(); i++) { bool isListCheck = false; bool isMultiListCheck = false; bool isHeaderCheck = false; HTTPHeader *targetheader = nullptr; bool state_result = false; ListMeta::list_result res; String target; String target2; String targetful; std::deque targetdq; switch (i->state) { case SB_STATE_SITEIN: isListCheck = true; target = cm.urldomain; target2 = target; targetful = cm.url; if (has_reverse_hosts(targetdq, cm)) isMultiListCheck = true; break; case SB_STATE_URLIN: isListCheck = true; target = cm.baseurl; target2 = cm.urldomain; targetful = cm.url; if (has_reverse_hosts(targetdq, cm)) isMultiListCheck = true; break; case SB_STATE_FULLURLIN: isListCheck = true; target = cm.baseurl; target2 = cm.urldomain; targetful = cm.url; if (has_reverse_hosts(targetdq, cm)) isMultiListCheck = true; break; case SB_STATE_SEARCHIN: if (cm.isSearch) { isListCheck = true; target = cm.search_words; } break; case SB_STATE_EMBEDDEDIN: if (!cm.deep_urls_checked) { cm.deep_urls = deep_urls(cm.baseurl, cm); if (cm.deep_urls.size() > 0) cm.hasEmbededURL = true; } if (cm.hasEmbededURL) { isMultiListCheck = true; targetdq = cm.deep_urls; target = "mutli"; } break; case SB_STATE_REFERERIN: isListCheck = true; target = cm.request_header->getReferer(); // needs spliting before?? target2 = target.getHostname(); break; case SB_STATE_HEADERIN: isHeaderCheck = true; targetheader = cm.request_header; break; case SB_STATE_RESHEADERIN: isHeaderCheck = true; targetheader = cm.response_header; break; case SB_STATE_CLIENTIN: isListCheck = true; target = cm.clientip; target2 = cm.clienthost; break; case SB_STATE_USERIN: isListCheck = true; cm.user.toLower(); target = cm.user; target2 = cm.clienthost; break; case SB_STATE_LISTEN_PORTIN: isListCheck = true; target = cm.listen_port; break; case SB_STATE_TIMEIN: isListCheck = true; break; case SB_STATE_USERAGENTIN: isListCheck = true; targetful = cm.request_header->userAgent(); // needs spliting before??target = ""; target2 = ""; break; case SB_STATE_EXTENSIONIN: target = cm.response_header->disposition(); if (target.length() > 4) isListCheck = true; target2 = ""; break; case SB_STATE_MIMEIN: target = cm.response_header->getContentType(); if (target.length() > 4) isListCheck = true; target2 = ""; break; case SB_STATE_CATEGORYIN: target = cm.main_category(); DEBUG_debug("CAT target is ", target); if (target.length() > 2) isListCheck = true; target2 = ""; break; case SB_STATE_CONNECT: state_result = cm.isconnect; break; case SB_STATE_GET: state_result = (cm.request_header->requestType() == "GET"); break; case SB_STATE_POST: state_result = (cm.request_header->requestType() == "POST"); break; case SB_STATE_EXCEPTIONSET: state_result = cm.isexception; break; case SB_STATE_SEMIEXCEPTIONSET: state_result = cm.issemiexception; break; case SB_STATE_GREYSET: state_result = cm.isGrey; break; case SB_STATE_BLOCKSET: state_result = cm.isBlocked; break; case SB_STATE_MITMSET: state_result = cm.ismitm; break; case SB_STATE_DONESET: state_result = cm.isdone; break; case SB_STATE_RETURNSET: state_result = cm.isReturn; break; case SB_STATE_REDIRECTSET: state_result = cm.urlredirect; break; case SB_STATE_VIRUSCHECKSET: state_result = !cm.noviruscheck; break; case SB_STATE_BYPASSSET: state_result = cm.isbypass; break; case SB_STATE_BYPASSALLOWEDSET: state_result = cm.isbypassallowed; break; case SB_STATE_INFECTIONBYPASSALLOWEDSET: state_result = cm.isinfectionbypassallowed; break; case SB_STATE_HASSNI: state_result = cm.hasSNI; break; case SB_STATE_TLS: state_result = cm.isTLS; break; case SB_STATE_SITEISIP: state_result = cm.isiphost; break; case SB_STATE_TRUE: state_result = true; break; } DEBUG_story("SB-Test ", " state: ", F->getState(i->state), " target: ", target, " target2: ", target2, " list_check: ", String(isListCheck), " targetfull: ", targetful, " isSearch: ", String(cm.isSearch), " state_result: ", String(state_result)); if (isHeaderCheck) { DEBUG_trace("HeaderCheck"); for (std::deque::iterator u = targetheader->header.begin(); u != targetheader->header.end(); u++) { // String t = *u; for (std::deque::iterator j = i->list_id_dq.begin(); j != i->list_id_dq.end(); j++) { DEBUG_story("checking ", j->name, " type ", j->type); if (LMeta->inList(*j, *u, res)) { //found state_result = true; if (i->isif) { cm.lastcategory = res.category; cm.whatIsNaughtyCategories = res.category; cm.message_no = res.mess_no; cm.log_message_no = res.log_mess_no; cm.lastmatch = res.match; cm.result = res.result; if (j->type == LIST_TYPE_REGEXP_REP) { *u = res.result; } if (o.log.anonymise_logs) cm.anon_user = true; if (res.anon_log) { cm.anon_url = true; } } DEBUG_story("SB lc", cm.lastcategory, " mess_no ", cm.message_no, " log_mess ", cm.log_message_no, " match ", res.match); break; } } if (state_result) break; } } if (isListCheck) { DEBUG_story("ListCheck:", i->list_name, " size:", i->list_id_dq.size()); for (std::deque::iterator j = i->list_id_dq.begin(); j != i->list_id_dq.end(); j++) { //ListMeta::list_result res; String t; if ((j->type >= LIST_TYPE_SITE) && (j->type < LIST_TYPE_URL)) { t = target2; } else if (j->type == LIST_TYPE_REGEXP_BOOL || j->type == LIST_TYPE_REGEXP_REP) { t = targetful; } else if ((j->type == LIST_TYPE_FILE_EXT) && target.contains("?")) { t = target.before("?"); // remove cgi part of url } else { t = target; } DEBUG_story("ListCheck ", j->name, "(type ", j->type, ")", " for ", t); if (cm.issiteonly && (j->type == LIST_TYPE_URL || j->type == LIST_TYPE_FILE_EXT)) continue; if (!(cm.isiphost) && j->type == LIST_TYPE_IPSITE) continue; if ((cm.isiphost) && j->type == LIST_TYPE_SITE && !o.lists.search_sitelist_for_ip) continue; // DEBUG_story("checking ", j->name, " type ", String(j->type)); if (LMeta->inList(*j, t, res)) { //found state_result = true; if (i->isif) { cm.result = res.result; } DEBUG_story("ListCheck lc", cm.lastcategory, " mess_no ", cm.message_no, " log_mess ", cm.log_message_no, " match ", res.match); break; } } DEBUG_debug("ListCheck:", i->list_name, " NOT FOUND"); } if (isMultiListCheck && !state_result) { DEBUG_trace("MultiListCheck"); for (std::deque::iterator u = targetdq.begin(); u != targetdq.end(); u++) { for (std::deque::iterator j = i->list_id_dq.begin(); j != i->list_id_dq.end(); j++) { //ListMeta::list_result res; String t; if ((j->type >= LIST_TYPE_SITE) && (j->type < LIST_TYPE_URL)) { t = u->urldomain; } else if (j->type == LIST_TYPE_REGEXP_BOOL || j->type == LIST_TYPE_REGEXP_REP) { t = u->fullurl; } else { t = u->baseurl; } if (u->is_siteonly && j->type == LIST_TYPE_URL) continue; if (!(u->site_is_ip) && j->type == LIST_TYPE_IPSITE) continue; if ((u->site_is_ip) && j->type == LIST_TYPE_SITE && !o.lists.search_sitelist_for_ip) continue; DEBUG_story("checking ", j->name, " type ", j->type, "Target ", t); if (LMeta->inList(*j, t, res)) { //found state_result = true; if (i->isif) { cm.result = res.result; } DEBUG_story("SB lc", cm.lastcategory, " mess_no ", cm.message_no, " log_mess ", cm.log_message_no, " match ", res.match); break; } } if (state_result) break; } } if (!i->isif) { state_result = !state_result; } DEBUG_story("SB-Result", " state: ", F->getState(i->state), " target: ", target, " target2: ", target2, " list_check: ", String(isListCheck), " targetfull: ", targetful, " isSearch: ", String(cm.isSearch), " state_result: ", String(state_result)); E2LOGGER_storytrace("SB: ", i->file_lineno, (i->isif ? " if(" : " ifnot("), F->getState(i->state), ",", i->list_name, ") is ", (state_result ? "true" : "false")); if (!state_result) { action_return = false; cm.isReturn = action_return; continue; // nothing to do so continue to next SB line } action_return = true; // if (i->mess_no > 0) cm.message_no = i->mess_no; // if (i->log_mess_no > 0) cm.log_message_no = i->log_mess_no; DEBUG_story("ACTION ", " lc: ", cm.lastcategory, " mess_no: ", String(cm.message_no), " log_mess: ", String(cm.log_message_no), " match: ", cm.whatIsNaughty, " action_id: ", F->getBIFunct(i->action_id)); if (i->action_id > SB_BI_FUNC_BASE) { // is built-in action switch (i->action_id) { case SB_FUNC_SETEXCEPTION: cm.isexception = true; cm.isGrey = false; cm.isBlocked = false; if (!cm.issemiexception) { // don't overwrite semi exception messages update_messages(cm, res); if (i->mess_no > 0) cm.message_no = i->mess_no; if (i->log_mess_no > 0) cm.log_message_no = i->log_mess_no; cm.whatIsNaughty = o.language_list.getTranslation(cm.message_no) + cm.lastmatch; if (cm.log_message_no == 0) cm.whatIsNaughtyLog = cm.whatIsNaughty; else cm.whatIsNaughtyLog = o.language_list.getTranslation(cm.log_message_no) + cm.lastmatch; cm.exceptioncat = cm.lastcategory; } cm.issemiexception = false; break; case SB_FUNC_SETSEMIEXCEPTION: cm.issemiexception = true; cm.isexception = false; cm.isGrey = false; cm.isBlocked = false; update_messages(cm, res); if (i->mess_no > 0) cm.message_no = i->mess_no; if (i->log_mess_no > 0) cm.log_message_no = i->log_mess_no; //cm.exceptionreason = o.language_list.getTranslation(cm.message_no); cm.whatIsNaughty = o.language_list.getTranslation(cm.message_no) + cm.lastmatch; if (cm.log_message_no == 0) cm.whatIsNaughtyLog = cm.whatIsNaughty; else cm.whatIsNaughtyLog = o.language_list.getTranslation(cm.log_message_no) + cm.lastmatch; cm.exceptioncat = cm.lastcategory; break; case SB_FUNC_UNSETSEMIEXCEPTION: cm.issemiexception = false; break; case SB_FUNC_SETGREY: cm.isGrey = true; cm.isexception = false; cm.issemiexception = false; cm.isBlocked = false; break; case SB_FUNC_SETBLOCK: cm.isBlocked = true; cm.isGrey = false; cm.isexception = false; cm.issemiexception = false; update_messages(cm, res); if (i->mess_no > 0) cm.message_no = i->mess_no; if (i->log_mess_no > 0) cm.log_message_no = i->log_mess_no; if (cm.message_no == 503) cm.whatIsNaughty = o.language_list.getTranslation(cm.message_no); else cm.whatIsNaughty = o.language_list.getTranslation(cm.message_no) + cm.get_lastmatch(); if (cm.log_message_no == 0) cm.whatIsNaughtyLog = o.language_list.getTranslation(cm.message_no) + cm.get_lastmatch(); else cm.whatIsNaughtyLog = o.language_list.getTranslation(cm.log_message_no) + cm.get_lastmatch(); cm.whatIsNaughtyCategories = cm.lastcategory; break; case SB_FUNC_SETMODURL: cm.urlmodified = true; cm.request_header->setURL(cm.result); cm.url = cm.result; cm.baseurl = cm.url; cm.baseurl.removeWhiteSpace(); cm.baseurl.toLower(); cm.baseurl.removePTP(); cm.logurl = cm.request_header->getLogUrl(false, cm.ismitm); cm.urld = cm.request_header->decode(cm.url); cm.urldomain = cm.url.getHostname(); cm.urldomain.toLower(); cm.connect_site = cm.urldomain; DEBUG_story("SB: URL modified to ", cm.url); break; case SB_FUNC_SETCONNECTSITE: cm.urlmodified = true; cm.logurl = cm.result; cm.connect_site = cm.result.getHostname(); DEBUG_story("SB: connect site changed to ", cm.connect_site); break; case SB_FUNC_SETLOGCAT: cm.logcategory = true; update_messages(cm, res); if (i->mess_no > 0) cm.message_no = i->mess_no; if (i->log_mess_no > 0) cm.log_message_no = i->log_mess_no; cm.whatIsNaughty = o.language_list.getTranslation(cm.message_no) + cm.lastmatch; if (cm.log_message_no == 0) cm.whatIsNaughtyLog = cm.whatIsNaughty; else cm.whatIsNaughtyLog = o.language_list.getTranslation(cm.log_message_no) + cm.lastmatch; cm.whatIsNaughtyCategories = cm.lastcategory; break; case SB_FUNC_SETREDIRECT: if (cm.result.size() > 0) { cm.request_header->redirect = cm.result; cm.urlredirect = true; } else { action_return = false; } break; case SB_FUNC_SETGOMITM: if (cm.ismitmcandidate && !cm.nomitm) { cm.gomitm = true; } else { action_return = false; } break; case SB_FUNC_SETNOMITM: cm.nomitm = true; cm.gomitm = false; cm.automitm = false; break; case SB_FUNC_SETAUTOMITM: if (!cm.nomitm) cm.automitm = true; break; case SB_FUNC_UNSETAUTOMITM: cm.automitm = false; break; case SB_FUNC_SETADDHEADER: cm.headeradded = true; cm.request_header->addHeader(cm.result); break; case SB_FUNC_SETMODHEADER: cm.headermodified = true; break; case SB_FUNC_SETNOCHECKCERT: cm.nocheckcert = true; break; case SB_FUNC_SETSEARCHTERM: if (cm.result.size() > 0) { cm.isSearch = true; cm.search_words = cm.result.sort_search(); cm.search_terms = cm.result; cm.search_terms.swapChar('+', ' '); }; break; case SB_FUNC_SETGODIRECT: cm.isdirect = true; break; case SB_FUNC_SETDONE: cm.isdone = true; break; case SB_FUNC_SETNOLOG: cm.nolog = true; break; case SB_FUNC_SETALERT: cm.alert = true; break; case SB_FUNC_SETGROUP: action_return = false; if (cm.result.size() > 0) { int g = cm.result.toInteger(); if (g > 0 && g <= o.filter.numfg) { cm.filtergroup = --g; cm.authrec->group_source = i->list_name; action_return = true; } }; break; case SB_FUNC_UNSETVIRUSCHECK: cm.noviruscheck = true; break; case SB_FUNC_UNSETBYPASS: cm.isbypass = false; cm.iscookiebypass = false; cm.isscanbypass = false; cm.isvirusbypass = false; cm.isexception = false; break; case SB_FUNC_UNSETBYPASSALLOW: if (cm.isbypassallowed) cm.isbypassallowed = false; else action_return = false; break; case SB_FUNC_UNSETINFECTIONBYPASSALLOW: if (cm.isinfectionbypassallowed) cm.isinfectionbypassallowed = false; else action_return = false; break; case SB_FUNC_SETTRUE: break; case SB_FUNC_SETFALSE: action_return = false; break; } // DEBUG_story("SB-ACTION: ", F->name, " : ", F->getBIFunct(i->action_id), " ", (action_return? "true" : "false" )); } else { // is SB defined function if (i->action_id > 0) { action_return = runFunct(i->action_id, cm); E2LOGGER_storytrace("SB:Resuming ", F->name, " in ", F->file_name); } } cm.isReturn = action_return; if (i->return_after_action) break; if (i->return_after_action_is_true && action_return) break; } E2LOGGER_storytrace("SB:Exiting ", F->getName(), " returned ", (action_return ? "true" : "false")); return action_return; } bool StoryBoard::setEntry(unsigned int index, String fname) { entrys[index] = getFunctID(fname); if (entrys[index] > 0) { return true; } return false; }; bool StoryBoard::runFunctEntry(unsigned int index, NaughtyFilter &cm) { DEBUG_trace(""); cm.isdone = false; // only has logical scope for a single call if (entrys[index] > 0) return runFunct(entrys[index], cm); else return false; }; std::deque StoryBoard::deep_urls(String &urld, NaughtyFilter &cm) { std::deque temp; String durl = urld; // Google web site translate checking // Format is domain with '.' replaced with '-' and '-' with '--' ending with .translate.goo // so www.merriam-webster.com -> www-merriam--webster-com.translate.goog if (cm.urldomain.endsWith(".translate.goog")) { // E2LOGGER_info("Found translate.goog ", durl ); String tem_dom = cm.urldomain.before("."); String new_url; while (tem_dom.contains("-")) { new_url += tem_dom.before("-"); tem_dom = tem_dom.after("-"); if (tem_dom.startsWith("-")) { // is a '--' so replace with '-' new_url += "-"; tem_dom = tem_dom.after("-"); } else { new_url += "."; // is a '-' so replace with '.' } // E2LOGGER_info(" GT embedd url is ", new_url); } new_url += tem_dom; // E2LOGGER_info(" GT embedd url is ", new_url); if (!cm.issiteonly) { //replace url part if needed new_url += "/"; new_url += urld.after("/"); } // E2LOGGER_info(" GT embedd url is ", new_url); // E2LOGGER_info(" GT tem_dom is ", tem_dom); url_rec t; t.baseurl = new_url; t.fullurl = new_url; t.urldomain = t.baseurl.getHostname(); if (t.baseurl == t.urldomain) t.is_siteonly = true; if (cm.isIPHostnameStrip(t.urldomain)) t.site_is_ip = true; temp.push_back(t); } while (durl.contains(":")) { durl = durl.after(":"); if (!durl.contains(".")) break; while (durl.startsWith(":'") || durl.startsWith("/")) { durl.lop(); } if (durl.size() > 5) { url_rec t; t.baseurl = durl; t.baseurl.removePTP(); t.fullurl = durl; if (durl.startsWith("http:") || durl.startsWith("https:")) durl = durl.after(":"); t.urldomain = t.baseurl.getHostname(); if (t.baseurl == t.urldomain) t.is_siteonly = true; if (cm.isIPHostnameStrip(t.urldomain)) t.site_is_ip = true; temp.push_back(t); } else { break; } } return temp; } // reverse DNS lookup on IP. be aware that this can return multiple results, unlike a standard lookup. std::deque StoryBoard::ipToHostname(NaughtyFilter &cm) { std::deque result; const char *ip = cm.urldomain.c_str(); String urlp = cm.urld.after("/"); //struct in_addr address, **addrptr; struct in_addr address; if (inet_aton(ip, &address)) { // convert to in_addr struct hostent *answer; answer = gethostbyaddr((char *) &address, sizeof(address), AF_INET); if (answer) { // sucess in reverse dns url_rec t; t.urldomain = answer->h_name; t.baseurl = t.urldomain + "/" + urlp; t.fullurl = "http://" + t.baseurl; result.push_back(t); //for (addrptr = (struct in_addr **)answer->h_addr_list; *addrptr; addrptr++) { // result->push_back(String(inet_ntoa(**addrptr))); //} } } return result; } bool StoryBoard::has_reverse_hosts(std::deque &urec, NaughtyFilter &cm) { if (!(cm.isiphost && o.story.reverse_lookups)) return false; if (!cm.reverse_checked) { cm.reversedURLs = ipToHostname(cm); cm.reverse_checked = true; } if (cm.reversedURLs.size() > 0) { urec = cm.reversedURLs; return true; } return false; } void StoryBoard::update_messages(NaughtyFilter &cm, ListMeta::list_result &res) { cm.lastcategory = res.category; cm.whatIsNaughtyCategories = res.category; cm.message_no = res.mess_no; cm.log_message_no = res.log_mess_no; cm.lastmatch = res.match; cm.result = res.result; if (res.anon_log) { cm.anon_user = true; cm.anon_url = true; } }