| 1 |
// ListMeta - super-class for both item and phrase lists |
| 2 |
|
| 3 |
// For all support, instructions and copyright go to: |
| 4 |
// http://e2guardian.org/ |
| 5 |
// Released under the GPL v2, with the OpenSSL exception described in the README file. |
| 6 |
|
| 7 |
// INCLUDES |
| 8 |
|
| 9 |
#ifdef HAVE_CONFIG_H |
| 10 |
#include "e2config.h" |
| 11 |
#endif |
| 12 |
|
| 13 |
#include "ListContainer.hpp" |
| 14 |
#include "ListMeta.hpp" |
| 15 |
#include "OptionContainer.hpp" |
| 16 |
#include "RegExp.hpp" |
| 17 |
#include "Logger.hpp" |
| 18 |
|
| 19 |
#include <algorithm> |
| 20 |
#include <cstdlib> |
| 21 |
#include <cstdio> |
| 22 |
#include <ctime> |
| 23 |
#include <unistd.h> |
| 24 |
#include <iostream> |
| 25 |
#include <fstream> |
| 26 |
#include <sys/stat.h> |
| 27 |
#include <sys/time.h> |
| 28 |
#include <list> |
| 29 |
#include <vector> |
| 30 |
|
| 31 |
// GLOBALS |
| 32 |
|
| 33 |
extern OptionContainer o; |
| 34 |
|
| 35 |
// DEFINES |
| 36 |
|
| 37 |
// Constructor - set default values |
| 38 |
ListMeta::ListMeta() { |
| 39 |
} |
| 40 |
|
| 41 |
// delete the memory block when the class is destryed |
| 42 |
ListMeta::~ListMeta() { |
| 43 |
reset(); |
| 44 |
} |
| 45 |
|
| 46 |
String ListMeta::list_type(int type) { |
| 47 |
if (type > LIST_TYPE_ERROR || type < 0) |
| 48 |
type = LIST_TYPE_ERROR; |
| 49 |
return type_map[type]; |
| 50 |
} |
| 51 |
|
| 52 |
// clear & reset all values |
| 53 |
void ListMeta::reset() { |
| 54 |
for (std::vector<struct list_info>::iterator i = list_vec.begin(); i != list_vec.end(); i++) { |
| 55 |
o.lm.deRefList(i->list_ref); |
| 56 |
i->comp.clear(); |
| 57 |
i->reg_list_ref.clear(); |
| 58 |
} |
| 59 |
} |
| 60 |
|
| 61 |
bool ListMeta::load_type(int type, std::deque<String> &list) { |
| 62 |
unsigned int method_type = 0; |
| 63 |
switch (type) { |
| 64 |
case LIST_TYPE_IP : |
| 65 |
method_type = LIST_METHOD_IP; |
| 66 |
break; |
| 67 |
case LIST_TYPE_IPSITE : |
| 68 |
method_type = LIST_METHOD_IP; |
| 69 |
break; |
| 70 |
case LIST_TYPE_SITE : |
| 71 |
method_type = LIST_METHOD_READF_EWS; |
| 72 |
break; |
| 73 |
case LIST_TYPE_URL : |
| 74 |
method_type = LIST_METHOD_READF_SWS; |
| 75 |
break; |
| 76 |
case LIST_TYPE_SEARCH: |
| 77 |
method_type = LIST_METHOD_READF_SWS; |
| 78 |
break; |
| 79 |
case LIST_TYPE_REGEXP_BOOL: |
| 80 |
method_type = LIST_METHOD_REGEXP_BOOL; |
| 81 |
break; |
| 82 |
case LIST_TYPE_REGEXP_REP : |
| 83 |
method_type = LIST_METHOD_REGEXP_REPL; |
| 84 |
break; |
| 85 |
case LIST_TYPE_FILE_EXT: |
| 86 |
method_type = LIST_METHOD_READF_EWS; |
| 87 |
break; |
| 88 |
case LIST_TYPE_MIME: |
| 89 |
method_type = LIST_METHOD_READF_SWS; |
| 90 |
break; |
| 91 |
case LIST_TYPE_TIME: |
| 92 |
method_type = LIST_METHOD_TIME; |
| 93 |
break; |
| 94 |
case LIST_TYPE_MAP: |
| 95 |
method_type = LIST_METHOD_MAP; |
| 96 |
break; |
| 97 |
case LIST_TYPE_IPMAP: |
| 98 |
method_type = LIST_METHOD_IPMAP; |
| 99 |
break; |
| 100 |
case LIST_TYPE_CATEGORY: |
| 101 |
method_type = LIST_METHOD_READF_SWS; |
| 102 |
break; |
| 103 |
// PhraseList types to be added |
| 104 |
} |
| 105 |
bool errors = false; |
| 106 |
int dq_size = list.size(); |
| 107 |
for (int i = dq_size - 1; i > -1; i--) { // search backward thru list |
| 108 |
// parse line |
| 109 |
String t; |
| 110 |
t = list[i]; |
| 111 |
DEBUG_debug("reading ", t); |
| 112 |
String nm, fpath, pwd; |
| 113 |
bool anonlog = false; |
| 114 |
bool sitewild = true; |
| 115 |
unsigned int m_no = 0, log_m_no = 0; |
| 116 |
t.removeWhiteSpace(); |
| 117 |
t.removeChar('\''); |
| 118 |
t += ","; |
| 119 |
while (t.length() > 0) { |
| 120 |
if (t.startsWith("name=")) { |
| 121 |
nm = t.after("=").before(","); |
| 122 |
} else if (t.startsWith("messageno=")) { |
| 123 |
m_no = t.after("=").before(",").toInteger(); |
| 124 |
} else if (t.startsWith("logmessageno=")) { |
| 125 |
log_m_no = t.after("=").before(",").toInteger(); |
| 126 |
} else if (t.startsWith("path=")) { |
| 127 |
fpath = t.after("=").before(","); |
| 128 |
} else if (t.startsWith("listdir=")) { |
| 129 |
pwd = t.after("=").before(","); |
| 130 |
} else if (t.startsWith("anonlog=true")) { |
| 131 |
anonlog = true; |
| 132 |
} else if (t.startsWith("sitewild=false")) { |
| 133 |
sitewild = false; |
| 134 |
} |
| 135 |
t = t.after(","); |
| 136 |
t.removeWhiteSpace(); |
| 137 |
} |
| 138 |
if (list_exists(nm, type)) { |
| 139 |
E2LOGGER_info("List name '", nm, "' of this type already defined - ignoring", t); |
| 140 |
errors = true; |
| 141 |
continue; |
| 142 |
} |
| 143 |
|
| 144 |
list_info rec; |
| 145 |
rec.type = type; |
| 146 |
rec.method_type = method_type; |
| 147 |
rec.name = nm; |
| 148 |
rec.pwd = pwd; |
| 149 |
rec.mess_no = m_no; |
| 150 |
rec.anon_log = anonlog; |
| 151 |
rec.site_wild = sitewild; |
| 152 |
if (log_m_no) { |
| 153 |
rec.log_mess_no = log_m_no; |
| 154 |
} else { |
| 155 |
rec.log_mess_no = m_no; |
| 156 |
} |
| 157 |
|
| 158 |
DEBUG_debug("List name = ", nm, " m_no=", m_no, " log_m_no=", rec.log_mess_no, " path=", fpath); |
| 159 |
|
| 160 |
switch (method_type) { |
| 161 |
case LIST_METHOD_IP: |
| 162 |
if (readFile(fpath.toCharArray(), pwd.toCharArray(), &rec.list_ref, false, nm.toCharArray(), true)) { |
| 163 |
list_vec.push_back(rec); |
| 164 |
if (o.lm.l[rec.list_ref]->read_errors) errors = true; |
| 165 |
} else { |
| 166 |
E2LOGGER_error("Unable to read ", fpath); |
| 167 |
errors = true; |
| 168 |
}; |
| 169 |
break; |
| 170 |
case LIST_METHOD_IPMAP: |
| 171 |
if (readFile(fpath.toCharArray(), pwd.toCharArray(), &rec.list_ref, false, nm.toCharArray(), true, false, true)) { |
| 172 |
list_vec.push_back(rec); |
| 173 |
if (o.lm.l[rec.list_ref]->read_errors) errors = true; |
| 174 |
} else { |
| 175 |
E2LOGGER_error("Unable to read ", fpath); |
| 176 |
errors = true; |
| 177 |
}; |
| 178 |
break; |
| 179 |
case LIST_METHOD_MAP: |
| 180 |
if (readFile(fpath.toCharArray(), pwd.toCharArray(), &rec.list_ref, false, nm.toCharArray(), false, false, true)) { |
| 181 |
list_vec.push_back(rec); |
| 182 |
if (o.lm.l[rec.list_ref]->read_errors) errors = true; |
| 183 |
} else { |
| 184 |
E2LOGGER_error("Unable to read ", fpath); |
| 185 |
errors = true; |
| 186 |
}; |
| 187 |
break; |
| 188 |
case LIST_METHOD_TIME: |
| 189 |
if (readFile(fpath.toCharArray(), pwd.toCharArray(),&rec.list_ref, false, nm.toCharArray(), false, true)) { |
| 190 |
list_vec.push_back(rec); |
| 191 |
if (o.lm.l[rec.list_ref]->read_errors) errors = true; |
| 192 |
} else { |
| 193 |
E2LOGGER_error("Unable to read ", fpath); |
| 194 |
errors = true; |
| 195 |
}; |
| 196 |
break; |
| 197 |
case LIST_METHOD_READF_EWS : |
| 198 |
if (readFile(fpath.toCharArray(), pwd.toCharArray(), &rec.list_ref, false, nm.toCharArray())) { |
| 199 |
list_vec.push_back(rec); |
| 200 |
if (o.lm.l[rec.list_ref]->read_errors) errors = true; |
| 201 |
} else { |
| 202 |
E2LOGGER_error("Unable to read ", fpath); |
| 203 |
errors = true; |
| 204 |
}; |
| 205 |
break; |
| 206 |
case LIST_METHOD_READF_SWS : |
| 207 |
if (readFile(fpath.toCharArray(), pwd.toCharArray(), &rec.list_ref, true, nm.toCharArray())) { |
| 208 |
// test dump |
| 209 |
//DEBUG_debug(fpath, " read "); |
| 210 |
//o.lm.l[rec.list_ref]->dump_data(); // temp test only |
| 211 |
list_vec.push_back(rec); |
| 212 |
if (o.lm.l[rec.list_ref]->read_errors) errors = true; |
| 213 |
} else { |
| 214 |
E2LOGGER_error("Unable to read ", fpath); |
| 215 |
errors = true; |
| 216 |
}; |
| 217 |
break; |
| 218 |
case LIST_METHOD_REGEXP_BOOL : |
| 219 |
if (readRegExMatchFile(fpath.toCharArray(), pwd.toCharArray(), nm.toCharArray(), rec.list_ref, rec.comp, rec.source, |
| 220 |
rec.reg_list_ref)) { |
| 221 |
list_vec.push_back(rec); |
| 222 |
if (o.lm.l[rec.list_ref]->read_errors) errors = true; |
| 223 |
} else { |
| 224 |
E2LOGGER_error("Unable to read ", fpath); |
| 225 |
errors = true; |
| 226 |
}; |
| 227 |
break; |
| 228 |
case LIST_METHOD_REGEXP_REPL : |
| 229 |
if (readRegExReplacementFile(fpath.toCharArray(), pwd.toCharArray(),nm.toCharArray(), rec.list_ref, rec.replace, |
| 230 |
rec.comp)) { |
| 231 |
list_vec.push_back(rec); |
| 232 |
if (o.lm.l[rec.list_ref]->read_errors) errors = true; |
| 233 |
} else { |
| 234 |
E2LOGGER_error("Unable to read ", fpath); |
| 235 |
errors = true; |
| 236 |
}; |
| 237 |
break; |
| 238 |
} |
| 239 |
} |
| 240 |
if (errors) { |
| 241 |
return false; |
| 242 |
} |
| 243 |
// if (errors && o.lists.abort_on_missing_list) return false; |
| 244 |
return true; |
| 245 |
} |
| 246 |
|
| 247 |
|
| 248 |
bool ListMeta::list_exists(String name, int type) { |
| 249 |
if (findList(name, (int) type).name != "") |
| 250 |
return true; |
| 251 |
else |
| 252 |
return false; |
| 253 |
} |
| 254 |
|
| 255 |
ListMeta::list_info ListMeta::findList(String name, int tp) { |
| 256 |
list_info t; |
| 257 |
list_info *tptr; |
| 258 |
tptr = findListPtr(name, tp); |
| 259 |
if (tptr) { |
| 260 |
t = *tptr; |
| 261 |
} else |
| 262 |
t.list_ref = 0; |
| 263 |
return t; |
| 264 |
} |
| 265 |
|
| 266 |
ListMeta::list_info *ListMeta::findListPtr(String name, int tp) { |
| 267 |
list_info *t = nullptr; |
| 268 |
unsigned int type = (unsigned int) tp; |
| 269 |
//DEBUG_debug("Looking for ", name, " type ", type, " in listmeta"); |
| 270 |
for (std::vector<struct list_info>::iterator i = list_vec.begin(); i != list_vec.end(); i++) { |
| 271 |
if (i->name == name && i->type == type) { |
| 272 |
//DEBUG_debug("Found ", i->name, " type ", i->type, " in listmeta"); |
| 273 |
t = &(*i); |
| 274 |
return t; |
| 275 |
} |
| 276 |
} |
| 277 |
//DEBUG_debug("Not Found ", name, " type ", type, " in listmeta"); |
| 278 |
return t; |
| 279 |
} |
| 280 |
|
| 281 |
unsigned int ListMeta::findListId(String name, int type) { |
| 282 |
list_info t; |
| 283 |
t.list_ref = 0; |
| 284 |
t = findList(name, type); |
| 285 |
return t.list_ref; |
| 286 |
} |
| 287 |
|
| 288 |
bool ListMeta::inList(String name, int type, String &tofind, list_result &res) { |
| 289 |
list_info info = findList(name, type); |
| 290 |
return inList(info, tofind, res); |
| 291 |
} |
| 292 |
|
| 293 |
bool ListMeta::inList(list_info &info, std::deque<String> &header, list_result &res) { |
| 294 |
// this is only used for checking headers |
| 295 |
if (info.name == "") return false; |
| 296 |
int type = info.type; |
| 297 |
//char *match; |
| 298 |
switch (type) { |
| 299 |
case LIST_TYPE_REGEXP_BOOL : { |
| 300 |
int rc = inHeaderRegExp(info, header, res, res.category); |
| 301 |
if (rc == -1) { |
| 302 |
return false; |
| 303 |
} else { |
| 304 |
res.category = o.lm.l[info.reg_list_ref[rc]]->category; |
| 305 |
res.match = info.source[rc]; |
| 306 |
res.mess_no = info.mess_no; |
| 307 |
res.log_mess_no = info.log_mess_no; |
| 308 |
res.anon_log = info.anon_log; |
| 309 |
return true; |
| 310 |
} |
| 311 |
} |
| 312 |
case LIST_TYPE_REGEXP_REP: { |
| 313 |
if (info.comp.size() == 0) |
| 314 |
return false; |
| 315 |
if (headerRegExpReplace(info, header, res)) { |
| 316 |
res.mess_no = info.mess_no; |
| 317 |
res.log_mess_no = info.log_mess_no; |
| 318 |
res.anon_log = info.anon_log; |
| 319 |
return true; |
| 320 |
} |
| 321 |
return false; |
| 322 |
} |
| 323 |
} |
| 324 |
return false; |
| 325 |
} |
| 326 |
|
| 327 |
bool ListMeta::inList(list_info &info, String &tofind, list_result &res) { |
| 328 |
if (info.name == "") return false; |
| 329 |
int type = info.type; |
| 330 |
//const char *match; |
| 331 |
bool match; |
| 332 |
// String match; |
| 333 |
switch (type) { |
| 334 |
case LIST_TYPE_IP: |
| 335 |
match = o.lm.l[info.list_ref]->findInList(tofind.toCharArray(), res.category, res.match, res.result); |
| 336 |
if (!match) { |
| 337 |
return false; |
| 338 |
} else { |
| 339 |
// res.match = match; |
| 340 |
res.mess_no = info.mess_no; |
| 341 |
res.log_mess_no = info.log_mess_no; |
| 342 |
res.anon_log = info.anon_log; |
| 343 |
return true; |
| 344 |
} |
| 345 |
break; |
| 346 |
case LIST_TYPE_IPMAP: |
| 347 |
case LIST_TYPE_MAP: |
| 348 |
match = o.lm.l[info.list_ref]->findInList(tofind.toCharArray(), res.category, res.match, res.result); |
| 349 |
if (!match) { |
| 350 |
return false; |
| 351 |
} else { |
| 352 |
// res.result = match; |
| 353 |
res.mess_no = info.mess_no; |
| 354 |
res.log_mess_no = info.log_mess_no; |
| 355 |
res.anon_log = info.anon_log; |
| 356 |
return true; |
| 357 |
} |
| 358 |
break; |
| 359 |
case LIST_TYPE_TIME: |
| 360 |
if (o.lm.l[info.list_ref]->isNowInTimelist()) { |
| 361 |
res.mess_no = info.mess_no; |
| 362 |
res.log_mess_no = info.log_mess_no; |
| 363 |
res.anon_log = info.anon_log; |
| 364 |
return true; |
| 365 |
} |
| 366 |
return false; |
| 367 |
break; |
| 368 |
case LIST_TYPE_IPSITE: |
| 369 |
match = o.lm.l[info.list_ref]->findInList(tofind.toCharArray(), res.category, res.match, res.result); |
| 370 |
if (!match) { |
| 371 |
return false; |
| 372 |
} else { |
| 373 |
//res.match = match; |
| 374 |
res.mess_no = info.mess_no; |
| 375 |
res.log_mess_no = info.log_mess_no; |
| 376 |
res.anon_log = info.anon_log; |
| 377 |
return true; |
| 378 |
} |
| 379 |
break; |
| 380 |
case LIST_TYPE_SITE : |
| 381 |
match = inSiteList(tofind, info.list_ref, res.category,info.site_wild, res.match, res.result); |
| 382 |
if (!match) { |
| 383 |
return false; |
| 384 |
} else { |
| 385 |
//res.match = match; |
| 386 |
res.mess_no = info.mess_no; |
| 387 |
res.log_mess_no = info.log_mess_no; |
| 388 |
res.anon_log = info.anon_log; |
| 389 |
return true; |
| 390 |
} |
| 391 |
break; |
| 392 |
case LIST_TYPE_URL: |
| 393 |
match = inURLList(tofind, info.list_ref, res.category,info.site_wild, res.match, res.result); |
| 394 |
if (!match) { |
| 395 |
return false; |
| 396 |
} else { |
| 397 |
//res.match = match; |
| 398 |
res.mess_no = info.mess_no; |
| 399 |
res.log_mess_no = info.log_mess_no; |
| 400 |
res.anon_log = info.anon_log; |
| 401 |
return true; |
| 402 |
} |
| 403 |
break; |
| 404 |
case LIST_TYPE_SEARCH : |
| 405 |
match = inSearchList(tofind, info.list_ref, res.category, res.match, res.result); |
| 406 |
if (!match) { |
| 407 |
return false; |
| 408 |
} else { |
| 409 |
// res.match = match; |
| 410 |
res.mess_no = info.mess_no; |
| 411 |
res.log_mess_no = info.log_mess_no; |
| 412 |
res.anon_log = info.anon_log; |
| 413 |
return true; |
| 414 |
} |
| 415 |
break; |
| 416 |
case LIST_TYPE_MIME: |
| 417 |
case LIST_TYPE_CATEGORY: |
| 418 |
match = o.lm.l[info.list_ref]->findInList(tofind.toCharArray(), res.category, res.match, res.result); |
| 419 |
if (!match) { |
| 420 |
return false; |
| 421 |
} else { |
| 422 |
// res.match = match; |
| 423 |
res.mess_no = info.mess_no; |
| 424 |
res.log_mess_no = info.log_mess_no; |
| 425 |
res.anon_log = info.anon_log; |
| 426 |
return true; |
| 427 |
} |
| 428 |
break; |
| 429 |
case LIST_TYPE_FILE_EXT: |
| 430 |
match = o.lm.l[info.list_ref]->findEndsWith(tofind.toCharArray(), res.category, res.match); |
| 431 |
if (!match) { |
| 432 |
return false; |
| 433 |
} else { |
| 434 |
res.mess_no = info.mess_no; |
| 435 |
res.log_mess_no = info.log_mess_no; |
| 436 |
res.anon_log = info.anon_log; |
| 437 |
return true; |
| 438 |
} |
| 439 |
break; |
| 440 |
case LIST_TYPE_REGEXP_BOOL : { |
| 441 |
int rc = inRegExpURLList(tofind, info.comp, info.reg_list_ref, info.list_ref, res.category); |
| 442 |
if (rc == -1) { |
| 443 |
return false; |
| 444 |
} else { |
| 445 |
res.category = o.lm.l[info.reg_list_ref[rc]]->category; |
| 446 |
res.match = info.source[rc]; |
| 447 |
res.mess_no = info.mess_no; |
| 448 |
res.log_mess_no = info.log_mess_no; |
| 449 |
res.anon_log = info.anon_log; |
| 450 |
return true; |
| 451 |
} |
| 452 |
} |
| 453 |
break; |
| 454 |
case LIST_TYPE_REGEXP_REP: { |
| 455 |
if (info.comp.size() == 0) |
| 456 |
return false; |
| 457 |
String modified = tofind; |
| 458 |
if (regExp(modified, info.comp, info.replace)) { |
| 459 |
res.result = modified; |
| 460 |
res.mess_no = info.mess_no; |
| 461 |
res.log_mess_no = info.log_mess_no; |
| 462 |
res.anon_log = info.anon_log; |
| 463 |
return true; |
| 464 |
} |
| 465 |
return false; |
| 466 |
} |
| 467 |
break; |
| 468 |
} |
| 469 |
return false; |
| 470 |
} |
| 471 |
|
| 472 |
// read in the given file, write the list's ID into the given identifier, |
| 473 |
// sort using startsWith or endsWith depending on sortsw, and create a cache file if desired. |
| 474 |
// listname is used in error messages. |
| 475 |
bool ListMeta::readFile(const char *filename, const char *pwd, unsigned int *whichlist, bool sortsw, const char *listname, bool isip, bool istime, bool ismap) { |
| 476 |
DEBUG_config("read File: ", filename); |
| 477 |
if (strlen(filename) < 3) { |
| 478 |
E2LOGGER_error("Required Listname ", listname, " is not defined"); |
| 479 |
return false; |
| 480 |
} |
| 481 |
int res = o.lm.newItemList(filename, pwd, sortsw, 1, true, isip, istime, ismap); |
| 482 |
if (res < 0) { |
| 483 |
E2LOGGER_error("Error opening ", listname ); |
| 484 |
return false; |
| 485 |
} |
| 486 |
(*whichlist) = (unsigned) res; |
| 487 |
if (!(*o.lm.l[(*whichlist)]).used) { |
| 488 |
if(!istime) { |
| 489 |
if (sortsw) |
| 490 |
(*o.lm.l[(*whichlist)]).doSort(true); |
| 491 |
else |
| 492 |
(*o.lm.l[(*whichlist)]).doSort(false); |
| 493 |
} |
| 494 |
(*o.lm.l[(*whichlist)]).used = true; |
| 495 |
} |
| 496 |
return true; |
| 497 |
} |
| 498 |
|
| 499 |
|
| 500 |
bool ListMeta::inSiteList(String &urlp, unsigned int list, String &lastcategory, bool &site_wild, String &match, String &result) { |
| 501 |
|
| 502 |
String url = urlp; |
| 503 |
bool i; |
| 504 |
if (site_wild) { |
| 505 |
while (url.contains(".")) { |
| 506 |
i = (*o.lm.l[list]).findInList(url.toCharArray(), lastcategory, match, result); |
| 507 |
if (i) { |
| 508 |
return i; // exact match |
| 509 |
} |
| 510 |
url = url.after("."); // check for being in higher level domains |
| 511 |
} |
| 512 |
} |
| 513 |
if ((!url.contains(".")) && url.length() > 1) { // allows matching of .tld |
| 514 |
url = "." + url; |
| 515 |
} |
| 516 |
if(url.length() > 2) |
| 517 |
{ |
| 518 |
i = (*o.lm.l[list]).findInList(url.toCharArray(), lastcategory, match, result); |
| 519 |
if (i) { |
| 520 |
return i; // exact match |
| 521 |
} |
| 522 |
} |
| 523 |
return false; |
| 524 |
} |
| 525 |
|
| 526 |
bool ListMeta::inSearchList(String &words, unsigned int list, String &lastcategory, String &match, String &result) { |
| 527 |
bool i = (*o.lm.l[list]).findInList(words.toCharArray(), lastcategory, match, result); |
| 528 |
if (i) { |
| 529 |
return i; // exact match |
| 530 |
} |
| 531 |
return false; |
| 532 |
} |
| 533 |
|
| 534 |
|
| 535 |
// look in given URL list for given URL |
| 536 |
bool ListMeta::inURLList(String &urlp, unsigned int list, String &lc, bool &site_wild, String &match, String &result) { |
| 537 |
String url = urlp; |
| 538 |
unsigned int fl; |
| 539 |
//char *i; |
| 540 |
bool i; |
| 541 |
String foundurl; |
| 542 |
DEBUG_debug("inURLList: ", url); |
| 543 |
// url.removeWhiteSpace(); // just in case of weird browser crap |
| 544 |
// url.toLower(); |
| 545 |
// url.removePTP(); // chop off the ht(f)tp(s):// |
| 546 |
if (url.contains("/")) { |
| 547 |
String tpath("/"); |
| 548 |
tpath += url.after("/"); |
| 549 |
url = url.before("/"); |
| 550 |
tpath.hexDecode(); |
| 551 |
tpath.realPath(); |
| 552 |
url += tpath; // will resolve ../ and %2e2e/ and // etc |
| 553 |
} |
| 554 |
if (url.endsWith("/")) { |
| 555 |
url.chop(); // chop off trailing / if any |
| 556 |
} |
| 557 |
|
| 558 |
DEBUG_debug("inURLList (processed): ", url); |
| 559 |
|
| 560 |
while (url.before("/").contains(".")) { |
| 561 |
i = (*o.lm.l[list]).findStartsWith(url.c_str(), lc, match); |
| 562 |
if (i) { |
| 563 |
foundurl = match; |
| 564 |
fl = foundurl.length(); |
| 565 |
DEBUG_debug("foundurl: ", foundurl, foundurl.length()); |
| 566 |
DEBUG_debug("url: ", url, fl); |
| 567 |
if (url.length() > fl) { |
| 568 |
if (url[fl] == '/' || url[fl] == '?' || url[fl] == '&' || url[fl] == '=') { |
| 569 |
return true; // matches /blah/ or /blah/foo but not /blahfoo |
| 570 |
} |
| 571 |
} else { |
| 572 |
return true;// exact match |
| 573 |
} |
| 574 |
} |
| 575 |
if (!site_wild) |
| 576 |
break; |
| 577 |
url = url.after("."); // check for being in higher level domains |
| 578 |
} |
| 579 |
return false; |
| 580 |
} |
| 581 |
|
| 582 |
bool ListMeta::isIPHostname(String url) { |
| 583 |
RegResult Rre; |
| 584 |
if (!isiphost.match(url.toCharArray(), Rre)) { |
| 585 |
return true; |
| 586 |
} |
| 587 |
return false; |
| 588 |
} |
| 589 |
|
| 590 |
bool ListMeta::precompileregexps() { |
| 591 |
if (!isiphost.comp(".*[a-z|A-Z].*")) { |
| 592 |
E2LOGGER_error("Error compiling RegExp isiphost."); |
| 593 |
return false; |
| 594 |
} |
| 595 |
|
| 596 |
return true; |
| 597 |
} |
| 598 |
|
| 599 |
// read regexp url list |
| 600 |
bool ListMeta::readRegExMatchFile(const char *filename,const char *list_pwd, const char *listname, unsigned int &listref, |
| 601 |
std::deque<RegExp> &list_comp, std::deque<String> &list_source, |
| 602 |
std::deque<unsigned int> &list_ref) { |
| 603 |
int result = o.lm.newItemList(filename, list_pwd, true, 32, true); |
| 604 |
if (result < 0) { |
| 605 |
E2LOGGER_error("Error opening ", listname); |
| 606 |
return false; |
| 607 |
} |
| 608 |
listref = (unsigned) result; |
| 609 |
return compileRegExMatchFile(listref, list_comp, list_source, list_ref); |
| 610 |
} |
| 611 |
|
| 612 |
// NOTE TO SELF - MOVE TO LISTCONTAINER TO SOLVE FUE2E |
| 613 |
// compile regexp url list |
| 614 |
bool ListMeta::compileRegExMatchFile(unsigned int list, std::deque<RegExp> &list_comp, |
| 615 |
std::deque<String> &list_source, std::deque<unsigned int> &list_ref) { |
| 616 |
for (unsigned int i = 0; i < (*o.lm.l[list]).morelists.size(); i++) { |
| 617 |
if (!compileRegExMatchFile((*o.lm.l[list]).morelists[i], list_comp, list_source, list_ref)) { |
| 618 |
return false; |
| 619 |
} |
| 620 |
} |
| 621 |
RegExp r; |
| 622 |
bool rv = true; |
| 623 |
int len = (*o.lm.l[list]).getListLength(); |
| 624 |
String source; |
| 625 |
for (int i = 0; i < len; i++) { |
| 626 |
source = (*o.lm.l[list]).getItemAtInt(i).c_str(); |
| 627 |
rv = r.comp(source.toCharArray()); |
| 628 |
if (rv == false) { |
| 629 |
E2LOGGER_error("Error compiling regexp:", source); |
| 630 |
return false; |
| 631 |
} |
| 632 |
list_comp.push_back(r); |
| 633 |
list_source.push_back(source); |
| 634 |
list_ref.push_back(list); |
| 635 |
} |
| 636 |
(*o.lm.l[list]).used = true; |
| 637 |
return true; |
| 638 |
} |
| 639 |
|
| 640 |
// content and URL regular expression replacement files |
| 641 |
bool ListMeta::readRegExReplacementFile(const char *filename, const char *list_pwd, const char *listname, unsigned int &listid, |
| 642 |
std::deque<String> &list_rep, std::deque<RegExp> &list_comp) { |
| 643 |
int result = o.lm.newItemList(filename,list_pwd, true, 32, true); |
| 644 |
if (result < 0) { |
| 645 |
E2LOGGER_error("Error opening ", listname); |
| 646 |
return false; |
| 647 |
} |
| 648 |
listid = (unsigned) result; |
| 649 |
if (!(*o.lm.l[listid]).used) { |
| 650 |
//(*o.lm.l[listid]).doSort(true); |
| 651 |
(*o.lm.l[listid]).used = true; |
| 652 |
} |
| 653 |
RegExp r; |
| 654 |
bool rv = true; |
| 655 |
String regexp; |
| 656 |
String replacement; |
| 657 |
for (int i = 0; i < (*o.lm.l[listid]).getListLength(); i++) { |
| 658 |
regexp = (*o.lm.l[listid]).getItemAtInt(i).c_str(); |
| 659 |
replacement = regexp.after("\"->\""); |
| 660 |
while (!replacement.endsWith("\"")) { |
| 661 |
if (replacement.length() < 2) { |
| 662 |
break; |
| 663 |
} |
| 664 |
replacement.chop(); |
| 665 |
} |
| 666 |
replacement.chop(); |
| 667 |
regexp = regexp.after("\"").before("\"->\""); |
| 668 |
if (regexp.length() < 1) { // allow replace with nothing |
| 669 |
continue; |
| 670 |
} |
| 671 |
rv = r.comp(regexp.toCharArray()); |
| 672 |
if (rv == false) { |
| 673 |
E2LOGGER_error("Error compiling regexp: ", (*o.lm.l[listid]).getItemAtInt(i) ); |
| 674 |
return false; |
| 675 |
} |
| 676 |
list_comp.push_back(r); |
| 677 |
list_rep.push_back(replacement); |
| 678 |
} |
| 679 |
return true; |
| 680 |
} |
| 681 |
|
| 682 |
// is this URL in the given regexp URL list? |
| 683 |
int ListMeta::inRegExpURLList(String &urlin, std::deque<RegExp> &list_comp, std::deque<unsigned int> &list_ref, |
| 684 |
unsigned int list, String &lastcategory) { |
| 685 |
|
| 686 |
DEBUG_regexp("inRegExpURLList: ", urlin); |
| 687 |
// check parent list's time limit |
| 688 |
if (o.lm.l[list]->isNow()) { |
| 689 |
RegResult Rre; |
| 690 |
String url = urlin; |
| 691 |
url.removeWhiteSpace(); // just in case of weird browser crap |
| 692 |
url.toLower(); |
| 693 |
|
| 694 |
// whilst it would be nice to have regexes be able to match the PTP, |
| 695 |
// it has been assumed for too long that the URL string does not start with one, |
| 696 |
// and we don't want to break regexes that look explicitly for the start of |
| 697 |
// the string. changes here have therefore been reverted. 2005-12-07 |
| 698 |
url.removePTP(); |
| 699 |
if (url.contains("/")) { |
| 700 |
String tpath("/"); |
| 701 |
tpath += url.after("/"); |
| 702 |
url = url.before("/"); |
| 703 |
tpath.hexDecode(); |
| 704 |
tpath.realPath(); |
| 705 |
url += tpath; // will resolve ../ and %2e2e/ and // etc |
| 706 |
} |
| 707 |
if (url.endsWith("/")) { |
| 708 |
url.chop(); // chop off trailing / if any |
| 709 |
} |
| 710 |
// re-add the PTP |
| 711 |
/*if (ptp.length() > 0) |
| 712 |
url = ptp + "//" + url;*/ |
| 713 |
DEBUG_regexp("inRegExpURLList (processed): ", url); |
| 714 |
unsigned int i = 0; |
| 715 |
for (std::deque<RegExp>::iterator j = list_comp.begin(); j != list_comp.end(); j++) { |
| 716 |
if (o.lm.l[list_ref[i]]->isNow()) { |
| 717 |
if (j->match(url.toCharArray(), Rre)) |
| 718 |
return i; |
| 719 |
} |
| 720 |
else |
| 721 |
DEBUG_regexp("Outside included regexp list's time limit"); |
| 722 |
i++; |
| 723 |
} |
| 724 |
} |
| 725 |
else { |
| 726 |
DEBUG_regexp("Outside top level regexp list's time limit"); |
| 727 |
} |
| 728 |
return -1; |
| 729 |
} |
| 730 |
|
| 731 |
// Does a regexp search and replace. |
| 732 |
// urlRegExp Code originally from from Ton Gorter 2004 |
| 733 |
bool ListMeta::regExp(String &line, std::deque<RegExp> ®exp_list, std::deque<String> &replacement_list) { |
| 734 |
RegExp *re; |
| 735 |
RegResult Rre; |
| 736 |
String replacement; |
| 737 |
String repstr; |
| 738 |
String newLine; |
| 739 |
bool linemodified = false; |
| 740 |
unsigned int i; |
| 741 |
unsigned int j, k; |
| 742 |
unsigned int s = regexp_list.size(); |
| 743 |
unsigned int matches, submatches; |
| 744 |
unsigned int match; |
| 745 |
unsigned int srcoff; |
| 746 |
unsigned int nextoffset; |
| 747 |
unsigned int matchlen; |
| 748 |
unsigned int oldlinelen; |
| 749 |
|
| 750 |
if ((line.empty()) || line.length() < 3) |
| 751 |
return false; |
| 752 |
|
| 753 |
// iterate over our list of precompiled regexes |
| 754 |
for (i = 0; i < s; i++) { |
| 755 |
newLine = ""; |
| 756 |
re = &(regexp_list[i]); |
| 757 |
if (re->match(line.toCharArray(), Rre)) { |
| 758 |
repstr = replacement_list[i]; |
| 759 |
matches = Rre.numberOfMatches(); |
| 760 |
|
| 761 |
srcoff = 0; |
| 762 |
|
| 763 |
for (j = 0; j < matches; j++) { |
| 764 |
nextoffset = Rre.offset(j); |
| 765 |
matchlen = Rre.length(j); |
| 766 |
|
| 767 |
// copy next chunk of unmodified data |
| 768 |
if (nextoffset > srcoff) { |
| 769 |
newLine += line.subString(srcoff, nextoffset - srcoff); |
| 770 |
srcoff = nextoffset; |
| 771 |
} |
| 772 |
|
| 773 |
// Count number of submatches (brackets) in replacement string |
| 774 |
for (submatches = 0; j + submatches + 1 < matches; submatches++) |
| 775 |
if (Rre.offset(j + submatches + 1) + Rre.length(j + submatches + 1) > srcoff + matchlen) |
| 776 |
break; |
| 777 |
|
| 778 |
// \1 and $1 replacement |
| 779 |
replacement = ""; |
| 780 |
for (k = 0; k < repstr.length(); k++) { |
| 781 |
// find \1..\9 and $1..$9 and fill them in with submatched strings |
| 782 |
if ((repstr[k] == '\\' || repstr[k] == '$') && repstr[k + 1] >= '1' && repstr[k + 1] <= '9') { |
| 783 |
match = repstr[++k] - '0'; |
| 784 |
if (match <= submatches) { |
| 785 |
replacement += Rre.result(j + match).c_str(); |
| 786 |
} |
| 787 |
} else { |
| 788 |
// unescape \\ and \$, and add non-backreference characters to string |
| 789 |
if (repstr[k] == '\\' && (repstr[k + 1] == '\\' || repstr[k + 1] == '$')) |
| 790 |
k++; |
| 791 |
replacement += repstr.subString(k, 1); |
| 792 |
} |
| 793 |
} |
| 794 |
|
| 795 |
// copy filled in replacement string |
| 796 |
newLine += replacement; |
| 797 |
srcoff += matchlen; |
| 798 |
j += submatches; |
| 799 |
} |
| 800 |
oldlinelen = line.length(); |
| 801 |
if (srcoff < oldlinelen) { |
| 802 |
newLine += line.subString(srcoff, oldlinelen - srcoff); |
| 803 |
} |
| 804 |
DEBUG_regexp("Line modified! (", line, " -> ", newLine, ")" ); |
| 805 |
// copy newLine into line and continue with other regexes |
| 806 |
line = newLine; |
| 807 |
linemodified = true; |
| 808 |
} |
| 809 |
} |
| 810 |
|
| 811 |
return linemodified; |
| 812 |
|
| 813 |
} |
| 814 |
|
| 815 |
bool ListMeta::headerRegExpReplace(ListMeta::list_info &listi, std::deque<String> &header, list_result &res) { |
| 816 |
// exit immediately if list is empty |
| 817 |
if (not listi.comp.size()) |
| 818 |
return false; |
| 819 |
bool result = false; |
| 820 |
for (std::deque<String>::iterator i = header.begin(); i != header.end(); i++) { |
| 821 |
DEBUG_regexp("Starting header reg exp replace: ", *i); |
| 822 |
bool chop = false; |
| 823 |
if (i->endsWith("\r")) { |
| 824 |
i->chop(); |
| 825 |
chop = true; |
| 826 |
} |
| 827 |
result |= regExp(*i, listi.comp, listi.replace); |
| 828 |
if (chop) |
| 829 |
i->append("\r"); |
| 830 |
} |
| 831 |
|
| 832 |
for (std::deque<String>::iterator i = header.begin(); i != header.end(); i++) |
| 833 |
DEBUG_regexp("Starting header reg exp replace result: ", *i); |
| 834 |
|
| 835 |
return result; |
| 836 |
} |
| 837 |
|
| 838 |
int ListMeta::inHeaderRegExp(list_info &listi, std::deque<String> &header, list_result &res, String &lastcategory) { |
| 839 |
// exit immediately if list is empty |
| 840 |
if (not listi.comp.size()) |
| 841 |
return false; |
| 842 |
int result = -1; |
| 843 |
for (std::deque<String>::iterator i = header.begin(); i != header.end(); i++) { |
| 844 |
DEBUG_regexp("Starting header reg exp check ", *i); |
| 845 |
bool chop = false; |
| 846 |
if (i->endsWith("\r")) { |
| 847 |
i->chop(); |
| 848 |
chop = true; |
| 849 |
} |
| 850 |
result = inRegExpURLList(*i, listi.comp, listi.reg_list_ref, listi.list_ref, lastcategory); |
| 851 |
if (chop) |
| 852 |
i->append("\r"); |
| 853 |
if (result > -1) { |
| 854 |
res.category = lastcategory; |
| 855 |
//res.match = TODO add the info |
| 856 |
break; |
| 857 |
} |
| 858 |
} |
| 859 |
return result; |
| 860 |
} |
| 861 |
|