Debian Codegraph @

src/ListMeta.cpp

raw
1 // ListMeta - super-class for both item and phrase lists
2
3 // For all support, instructions and copyright go to:
4 // http://e2guardian.org/
5 // Released under the GPL v2, with the OpenSSL exception described in the README file.
6
7 // INCLUDES
8
9 #ifdef HAVE_CONFIG_H
10 #include "e2config.h"
11 #endif
12
13 #include "ListContainer.hpp"
14 #include "ListMeta.hpp"
15 #include "OptionContainer.hpp"
16 #include "RegExp.hpp"
17 #include "Logger.hpp"
18
19 #include <algorithm>
20 #include <cstdlib>
21 #include <cstdio>
22 #include <ctime>
23 #include <unistd.h>
24 #include <iostream>
25 #include <fstream>
26 #include <sys/stat.h>
27 #include <sys/time.h>
28 #include <list>
29 #include <vector>
30
31 // GLOBALS
32
33 extern OptionContainer o;
34
35 // DEFINES
36
37 // Constructor - set default values
38 ListMeta::ListMeta() {
39 }
40
41 // delete the memory block when the class is destryed
42 ListMeta::~ListMeta() {
43 reset();
44 }
45
46 String ListMeta::list_type(int type) {
47 if (type > LIST_TYPE_ERROR || type < 0)
48 type = LIST_TYPE_ERROR;
49 return type_map[type];
50 }
51
52 // clear & reset all values
53 void ListMeta::reset() {
54 for (std::vector<struct list_info>::iterator i = list_vec.begin(); i != list_vec.end(); i++) {
55 o.lm.deRefList(i->list_ref);
56 i->comp.clear();
57 i->reg_list_ref.clear();
58 }
59 }
60
61 bool ListMeta::load_type(int type, std::deque<String> &list) {
62 unsigned int method_type = 0;
63 switch (type) {
64 case LIST_TYPE_IP :
65 method_type = LIST_METHOD_IP;
66 break;
67 case LIST_TYPE_IPSITE :
68 method_type = LIST_METHOD_IP;
69 break;
70 case LIST_TYPE_SITE :
71 method_type = LIST_METHOD_READF_EWS;
72 break;
73 case LIST_TYPE_URL :
74 method_type = LIST_METHOD_READF_SWS;
75 break;
76 case LIST_TYPE_SEARCH:
77 method_type = LIST_METHOD_READF_SWS;
78 break;
79 case LIST_TYPE_REGEXP_BOOL:
80 method_type = LIST_METHOD_REGEXP_BOOL;
81 break;
82 case LIST_TYPE_REGEXP_REP :
83 method_type = LIST_METHOD_REGEXP_REPL;
84 break;
85 case LIST_TYPE_FILE_EXT:
86 method_type = LIST_METHOD_READF_EWS;
87 break;
88 case LIST_TYPE_MIME:
89 method_type = LIST_METHOD_READF_SWS;
90 break;
91 case LIST_TYPE_TIME:
92 method_type = LIST_METHOD_TIME;
93 break;
94 case LIST_TYPE_MAP:
95 method_type = LIST_METHOD_MAP;
96 break;
97 case LIST_TYPE_IPMAP:
98 method_type = LIST_METHOD_IPMAP;
99 break;
100 case LIST_TYPE_CATEGORY:
101 method_type = LIST_METHOD_READF_SWS;
102 break;
103 // PhraseList types to be added
104 }
105 bool errors = false;
106 int dq_size = list.size();
107 for (int i = dq_size - 1; i > -1; i--) { // search backward thru list
108 // parse line
109 String t;
110 t = list[i];
111 DEBUG_debug("reading ", t);
112 String nm, fpath, pwd;
113 bool anonlog = false;
114 bool sitewild = true;
115 unsigned int m_no = 0, log_m_no = 0;
116 t.removeWhiteSpace();
117 t.removeChar('\'');
118 t += ",";
119 while (t.length() > 0) {
120 if (t.startsWith("name=")) {
121 nm = t.after("=").before(",");
122 } else if (t.startsWith("messageno=")) {
123 m_no = t.after("=").before(",").toInteger();
124 } else if (t.startsWith("logmessageno=")) {
125 log_m_no = t.after("=").before(",").toInteger();
126 } else if (t.startsWith("path=")) {
127 fpath = t.after("=").before(",");
128 } else if (t.startsWith("listdir=")) {
129 pwd = t.after("=").before(",");
130 } else if (t.startsWith("anonlog=true")) {
131 anonlog = true;
132 } else if (t.startsWith("sitewild=false")) {
133 sitewild = false;
134 }
135 t = t.after(",");
136 t.removeWhiteSpace();
137 }
138 if (list_exists(nm, type)) {
139 E2LOGGER_info("List name '", nm, "' of this type already defined - ignoring", t);
140 errors = true;
141 continue;
142 }
143
144 list_info rec;
145 rec.type = type;
146 rec.method_type = method_type;
147 rec.name = nm;
148 rec.pwd = pwd;
149 rec.mess_no = m_no;
150 rec.anon_log = anonlog;
151 rec.site_wild = sitewild;
152 if (log_m_no) {
153 rec.log_mess_no = log_m_no;
154 } else {
155 rec.log_mess_no = m_no;
156 }
157
158 DEBUG_debug("List name = ", nm, " m_no=", m_no, " log_m_no=", rec.log_mess_no, " path=", fpath);
159
160 switch (method_type) {
161 case LIST_METHOD_IP:
162 if (readFile(fpath.toCharArray(), pwd.toCharArray(), &rec.list_ref, false, nm.toCharArray(), true)) {
163 list_vec.push_back(rec);
164 if (o.lm.l[rec.list_ref]->read_errors) errors = true;
165 } else {
166 E2LOGGER_error("Unable to read ", fpath);
167 errors = true;
168 };
169 break;
170 case LIST_METHOD_IPMAP:
171 if (readFile(fpath.toCharArray(), pwd.toCharArray(), &rec.list_ref, false, nm.toCharArray(), true, false, true)) {
172 list_vec.push_back(rec);
173 if (o.lm.l[rec.list_ref]->read_errors) errors = true;
174 } else {
175 E2LOGGER_error("Unable to read ", fpath);
176 errors = true;
177 };
178 break;
179 case LIST_METHOD_MAP:
180 if (readFile(fpath.toCharArray(), pwd.toCharArray(), &rec.list_ref, false, nm.toCharArray(), false, false, true)) {
181 list_vec.push_back(rec);
182 if (o.lm.l[rec.list_ref]->read_errors) errors = true;
183 } else {
184 E2LOGGER_error("Unable to read ", fpath);
185 errors = true;
186 };
187 break;
188 case LIST_METHOD_TIME:
189 if (readFile(fpath.toCharArray(), pwd.toCharArray(),&rec.list_ref, false, nm.toCharArray(), false, true)) {
190 list_vec.push_back(rec);
191 if (o.lm.l[rec.list_ref]->read_errors) errors = true;
192 } else {
193 E2LOGGER_error("Unable to read ", fpath);
194 errors = true;
195 };
196 break;
197 case LIST_METHOD_READF_EWS :
198 if (readFile(fpath.toCharArray(), pwd.toCharArray(), &rec.list_ref, false, nm.toCharArray())) {
199 list_vec.push_back(rec);
200 if (o.lm.l[rec.list_ref]->read_errors) errors = true;
201 } else {
202 E2LOGGER_error("Unable to read ", fpath);
203 errors = true;
204 };
205 break;
206 case LIST_METHOD_READF_SWS :
207 if (readFile(fpath.toCharArray(), pwd.toCharArray(), &rec.list_ref, true, nm.toCharArray())) {
208 // test dump
209 //DEBUG_debug(fpath, " read ");
210 //o.lm.l[rec.list_ref]->dump_data(); // temp test only
211 list_vec.push_back(rec);
212 if (o.lm.l[rec.list_ref]->read_errors) errors = true;
213 } else {
214 E2LOGGER_error("Unable to read ", fpath);
215 errors = true;
216 };
217 break;
218 case LIST_METHOD_REGEXP_BOOL :
219 if (readRegExMatchFile(fpath.toCharArray(), pwd.toCharArray(), nm.toCharArray(), rec.list_ref, rec.comp, rec.source,
220 rec.reg_list_ref)) {
221 list_vec.push_back(rec);
222 if (o.lm.l[rec.list_ref]->read_errors) errors = true;
223 } else {
224 E2LOGGER_error("Unable to read ", fpath);
225 errors = true;
226 };
227 break;
228 case LIST_METHOD_REGEXP_REPL :
229 if (readRegExReplacementFile(fpath.toCharArray(), pwd.toCharArray(),nm.toCharArray(), rec.list_ref, rec.replace,
230 rec.comp)) {
231 list_vec.push_back(rec);
232 if (o.lm.l[rec.list_ref]->read_errors) errors = true;
233 } else {
234 E2LOGGER_error("Unable to read ", fpath);
235 errors = true;
236 };
237 break;
238 }
239 }
240 if (errors) {
241 return false;
242 }
243 // if (errors && o.lists.abort_on_missing_list) return false;
244 return true;
245 }
246
247
248 bool ListMeta::list_exists(String name, int type) {
249 if (findList(name, (int) type).name != "")
250 return true;
251 else
252 return false;
253 }
254
255 ListMeta::list_info ListMeta::findList(String name, int tp) {
256 list_info t;
257 list_info *tptr;
258 tptr = findListPtr(name, tp);
259 if (tptr) {
260 t = *tptr;
261 } else
262 t.list_ref = 0;
263 return t;
264 }
265
266 ListMeta::list_info *ListMeta::findListPtr(String name, int tp) {
267 list_info *t = nullptr;
268 unsigned int type = (unsigned int) tp;
269 //DEBUG_debug("Looking for ", name, " type ", type, " in listmeta");
270 for (std::vector<struct list_info>::iterator i = list_vec.begin(); i != list_vec.end(); i++) {
271 if (i->name == name && i->type == type) {
272 //DEBUG_debug("Found ", i->name, " type ", i->type, " in listmeta");
273 t = &(*i);
274 return t;
275 }
276 }
277 //DEBUG_debug("Not Found ", name, " type ", type, " in listmeta");
278 return t;
279 }
280
281 unsigned int ListMeta::findListId(String name, int type) {
282 list_info t;
283 t.list_ref = 0;
284 t = findList(name, type);
285 return t.list_ref;
286 }
287
288 bool ListMeta::inList(String name, int type, String &tofind, list_result &res) {
289 list_info info = findList(name, type);
290 return inList(info, tofind, res);
291 }
292
293 bool ListMeta::inList(list_info &info, std::deque<String> &header, list_result &res) {
294 // this is only used for checking headers
295 if (info.name == "") return false;
296 int type = info.type;
297 //char *match;
298 switch (type) {
299 case LIST_TYPE_REGEXP_BOOL : {
300 int rc = inHeaderRegExp(info, header, res, res.category);
301 if (rc == -1) {
302 return false;
303 } else {
304 res.category = o.lm.l[info.reg_list_ref[rc]]->category;
305 res.match = info.source[rc];
306 res.mess_no = info.mess_no;
307 res.log_mess_no = info.log_mess_no;
308 res.anon_log = info.anon_log;
309 return true;
310 }
311 }
312 case LIST_TYPE_REGEXP_REP: {
313 if (info.comp.size() == 0)
314 return false;
315 if (headerRegExpReplace(info, header, res)) {
316 res.mess_no = info.mess_no;
317 res.log_mess_no = info.log_mess_no;
318 res.anon_log = info.anon_log;
319 return true;
320 }
321 return false;
322 }
323 }
324 return false;
325 }
326
327 bool ListMeta::inList(list_info &info, String &tofind, list_result &res) {
328 if (info.name == "") return false;
329 int type = info.type;
330 //const char *match;
331 bool match;
332 // String match;
333 switch (type) {
334 case LIST_TYPE_IP:
335 match = o.lm.l[info.list_ref]->findInList(tofind.toCharArray(), res.category, res.match, res.result);
336 if (!match) {
337 return false;
338 } else {
339 // res.match = match;
340 res.mess_no = info.mess_no;
341 res.log_mess_no = info.log_mess_no;
342 res.anon_log = info.anon_log;
343 return true;
344 }
345 break;
346 case LIST_TYPE_IPMAP:
347 case LIST_TYPE_MAP:
348 match = o.lm.l[info.list_ref]->findInList(tofind.toCharArray(), res.category, res.match, res.result);
349 if (!match) {
350 return false;
351 } else {
352 // res.result = match;
353 res.mess_no = info.mess_no;
354 res.log_mess_no = info.log_mess_no;
355 res.anon_log = info.anon_log;
356 return true;
357 }
358 break;
359 case LIST_TYPE_TIME:
360 if (o.lm.l[info.list_ref]->isNowInTimelist()) {
361 res.mess_no = info.mess_no;
362 res.log_mess_no = info.log_mess_no;
363 res.anon_log = info.anon_log;
364 return true;
365 }
366 return false;
367 break;
368 case LIST_TYPE_IPSITE:
369 match = o.lm.l[info.list_ref]->findInList(tofind.toCharArray(), res.category, res.match, res.result);
370 if (!match) {
371 return false;
372 } else {
373 //res.match = match;
374 res.mess_no = info.mess_no;
375 res.log_mess_no = info.log_mess_no;
376 res.anon_log = info.anon_log;
377 return true;
378 }
379 break;
380 case LIST_TYPE_SITE :
381 match = inSiteList(tofind, info.list_ref, res.category,info.site_wild, res.match, res.result);
382 if (!match) {
383 return false;
384 } else {
385 //res.match = match;
386 res.mess_no = info.mess_no;
387 res.log_mess_no = info.log_mess_no;
388 res.anon_log = info.anon_log;
389 return true;
390 }
391 break;
392 case LIST_TYPE_URL:
393 match = inURLList(tofind, info.list_ref, res.category,info.site_wild, res.match, res.result);
394 if (!match) {
395 return false;
396 } else {
397 //res.match = match;
398 res.mess_no = info.mess_no;
399 res.log_mess_no = info.log_mess_no;
400 res.anon_log = info.anon_log;
401 return true;
402 }
403 break;
404 case LIST_TYPE_SEARCH :
405 match = inSearchList(tofind, info.list_ref, res.category, res.match, res.result);
406 if (!match) {
407 return false;
408 } else {
409 // res.match = match;
410 res.mess_no = info.mess_no;
411 res.log_mess_no = info.log_mess_no;
412 res.anon_log = info.anon_log;
413 return true;
414 }
415 break;
416 case LIST_TYPE_MIME:
417 case LIST_TYPE_CATEGORY:
418 match = o.lm.l[info.list_ref]->findInList(tofind.toCharArray(), res.category, res.match, res.result);
419 if (!match) {
420 return false;
421 } else {
422 // res.match = match;
423 res.mess_no = info.mess_no;
424 res.log_mess_no = info.log_mess_no;
425 res.anon_log = info.anon_log;
426 return true;
427 }
428 break;
429 case LIST_TYPE_FILE_EXT:
430 match = o.lm.l[info.list_ref]->findEndsWith(tofind.toCharArray(), res.category, res.match);
431 if (!match) {
432 return false;
433 } else {
434 res.mess_no = info.mess_no;
435 res.log_mess_no = info.log_mess_no;
436 res.anon_log = info.anon_log;
437 return true;
438 }
439 break;
440 case LIST_TYPE_REGEXP_BOOL : {
441 int rc = inRegExpURLList(tofind, info.comp, info.reg_list_ref, info.list_ref, res.category);
442 if (rc == -1) {
443 return false;
444 } else {
445 res.category = o.lm.l[info.reg_list_ref[rc]]->category;
446 res.match = info.source[rc];
447 res.mess_no = info.mess_no;
448 res.log_mess_no = info.log_mess_no;
449 res.anon_log = info.anon_log;
450 return true;
451 }
452 }
453 break;
454 case LIST_TYPE_REGEXP_REP: {
455 if (info.comp.size() == 0)
456 return false;
457 String modified = tofind;
458 if (regExp(modified, info.comp, info.replace)) {
459 res.result = modified;
460 res.mess_no = info.mess_no;
461 res.log_mess_no = info.log_mess_no;
462 res.anon_log = info.anon_log;
463 return true;
464 }
465 return false;
466 }
467 break;
468 }
469 return false;
470 }
471
472 // read in the given file, write the list's ID into the given identifier,
473 // sort using startsWith or endsWith depending on sortsw, and create a cache file if desired.
474 // listname is used in error messages.
475 bool ListMeta::readFile(const char *filename, const char *pwd, unsigned int *whichlist, bool sortsw, const char *listname, bool isip, bool istime, bool ismap) {
476 DEBUG_config("read File: ", filename);
477 if (strlen(filename) < 3) {
478 E2LOGGER_error("Required Listname ", listname, " is not defined");
479 return false;
480 }
481 int res = o.lm.newItemList(filename, pwd, sortsw, 1, true, isip, istime, ismap);
482 if (res < 0) {
483 E2LOGGER_error("Error opening ", listname );
484 return false;
485 }
486 (*whichlist) = (unsigned) res;
487 if (!(*o.lm.l[(*whichlist)]).used) {
488 if(!istime) {
489 if (sortsw)
490 (*o.lm.l[(*whichlist)]).doSort(true);
491 else
492 (*o.lm.l[(*whichlist)]).doSort(false);
493 }
494 (*o.lm.l[(*whichlist)]).used = true;
495 }
496 return true;
497 }
498
499
500 bool ListMeta::inSiteList(String &urlp, unsigned int list, String &lastcategory, bool &site_wild, String &match, String &result) {
501
502 String url = urlp;
503 bool i;
504 if (site_wild) {
505 while (url.contains(".")) {
506 i = (*o.lm.l[list]).findInList(url.toCharArray(), lastcategory, match, result);
507 if (i) {
508 return i; // exact match
509 }
510 url = url.after("."); // check for being in higher level domains
511 }
512 }
513 if ((!url.contains(".")) && url.length() > 1) { // allows matching of .tld
514 url = "." + url;
515 }
516 if(url.length() > 2)
517 {
518 i = (*o.lm.l[list]).findInList(url.toCharArray(), lastcategory, match, result);
519 if (i) {
520 return i; // exact match
521 }
522 }
523 return false;
524 }
525
526 bool ListMeta::inSearchList(String &words, unsigned int list, String &lastcategory, String &match, String &result) {
527 bool i = (*o.lm.l[list]).findInList(words.toCharArray(), lastcategory, match, result);
528 if (i) {
529 return i; // exact match
530 }
531 return false;
532 }
533
534
535 // look in given URL list for given URL
536 bool ListMeta::inURLList(String &urlp, unsigned int list, String &lc, bool &site_wild, String &match, String &result) {
537 String url = urlp;
538 unsigned int fl;
539 //char *i;
540 bool i;
541 String foundurl;
542 DEBUG_debug("inURLList: ", url);
543 // url.removeWhiteSpace(); // just in case of weird browser crap
544 // url.toLower();
545 // url.removePTP(); // chop off the ht(f)tp(s)://
546 if (url.contains("/")) {
547 String tpath("/");
548 tpath += url.after("/");
549 url = url.before("/");
550 tpath.hexDecode();
551 tpath.realPath();
552 url += tpath; // will resolve ../ and %2e2e/ and // etc
553 }
554 if (url.endsWith("/")) {
555 url.chop(); // chop off trailing / if any
556 }
557
558 DEBUG_debug("inURLList (processed): ", url);
559
560 while (url.before("/").contains(".")) {
561 i = (*o.lm.l[list]).findStartsWith(url.c_str(), lc, match);
562 if (i) {
563 foundurl = match;
564 fl = foundurl.length();
565 DEBUG_debug("foundurl: ", foundurl, foundurl.length());
566 DEBUG_debug("url: ", url, fl);
567 if (url.length() > fl) {
568 if (url[fl] == '/' || url[fl] == '?' || url[fl] == '&' || url[fl] == '=') {
569 return true; // matches /blah/ or /blah/foo but not /blahfoo
570 }
571 } else {
572 return true;// exact match
573 }
574 }
575 if (!site_wild)
576 break;
577 url = url.after("."); // check for being in higher level domains
578 }
579 return false;
580 }
581
582 bool ListMeta::isIPHostname(String url) {
583 RegResult Rre;
584 if (!isiphost.match(url.toCharArray(), Rre)) {
585 return true;
586 }
587 return false;
588 }
589
590 bool ListMeta::precompileregexps() {
591 if (!isiphost.comp(".*[a-z|A-Z].*")) {
592 E2LOGGER_error("Error compiling RegExp isiphost.");
593 return false;
594 }
595
596 return true;
597 }
598
599 // read regexp url list
600 bool ListMeta::readRegExMatchFile(const char *filename,const char *list_pwd, const char *listname, unsigned int &listref,
601 std::deque<RegExp> &list_comp, std::deque<String> &list_source,
602 std::deque<unsigned int> &list_ref) {
603 int result = o.lm.newItemList(filename, list_pwd, true, 32, true);
604 if (result < 0) {
605 E2LOGGER_error("Error opening ", listname);
606 return false;
607 }
608 listref = (unsigned) result;
609 return compileRegExMatchFile(listref, list_comp, list_source, list_ref);
610 }
611
612 // NOTE TO SELF - MOVE TO LISTCONTAINER TO SOLVE FUE2E
613 // compile regexp url list
614 bool ListMeta::compileRegExMatchFile(unsigned int list, std::deque<RegExp> &list_comp,
615 std::deque<String> &list_source, std::deque<unsigned int> &list_ref) {
616 for (unsigned int i = 0; i < (*o.lm.l[list]).morelists.size(); i++) {
617 if (!compileRegExMatchFile((*o.lm.l[list]).morelists[i], list_comp, list_source, list_ref)) {
618 return false;
619 }
620 }
621 RegExp r;
622 bool rv = true;
623 int len = (*o.lm.l[list]).getListLength();
624 String source;
625 for (int i = 0; i < len; i++) {
626 source = (*o.lm.l[list]).getItemAtInt(i).c_str();
627 rv = r.comp(source.toCharArray());
628 if (rv == false) {
629 E2LOGGER_error("Error compiling regexp:", source);
630 return false;
631 }
632 list_comp.push_back(r);
633 list_source.push_back(source);
634 list_ref.push_back(list);
635 }
636 (*o.lm.l[list]).used = true;
637 return true;
638 }
639
640 // content and URL regular expression replacement files
641 bool ListMeta::readRegExReplacementFile(const char *filename, const char *list_pwd, const char *listname, unsigned int &listid,
642 std::deque<String> &list_rep, std::deque<RegExp> &list_comp) {
643 int result = o.lm.newItemList(filename,list_pwd, true, 32, true);
644 if (result < 0) {
645 E2LOGGER_error("Error opening ", listname);
646 return false;
647 }
648 listid = (unsigned) result;
649 if (!(*o.lm.l[listid]).used) {
650 //(*o.lm.l[listid]).doSort(true);
651 (*o.lm.l[listid]).used = true;
652 }
653 RegExp r;
654 bool rv = true;
655 String regexp;
656 String replacement;
657 for (int i = 0; i < (*o.lm.l[listid]).getListLength(); i++) {
658 regexp = (*o.lm.l[listid]).getItemAtInt(i).c_str();
659 replacement = regexp.after("\"->\"");
660 while (!replacement.endsWith("\"")) {
661 if (replacement.length() < 2) {
662 break;
663 }
664 replacement.chop();
665 }
666 replacement.chop();
667 regexp = regexp.after("\"").before("\"->\"");
668 if (regexp.length() < 1) { // allow replace with nothing
669 continue;
670 }
671 rv = r.comp(regexp.toCharArray());
672 if (rv == false) {
673 E2LOGGER_error("Error compiling regexp: ", (*o.lm.l[listid]).getItemAtInt(i) );
674 return false;
675 }
676 list_comp.push_back(r);
677 list_rep.push_back(replacement);
678 }
679 return true;
680 }
681
682 // is this URL in the given regexp URL list?
683 int ListMeta::inRegExpURLList(String &urlin, std::deque<RegExp> &list_comp, std::deque<unsigned int> &list_ref,
684 unsigned int list, String &lastcategory) {
685
686 DEBUG_regexp("inRegExpURLList: ", urlin);
687 // check parent list's time limit
688 if (o.lm.l[list]->isNow()) {
689 RegResult Rre;
690 String url = urlin;
691 url.removeWhiteSpace(); // just in case of weird browser crap
692 url.toLower();
693
694 // whilst it would be nice to have regexes be able to match the PTP,
695 // it has been assumed for too long that the URL string does not start with one,
696 // and we don't want to break regexes that look explicitly for the start of
697 // the string. changes here have therefore been reverted. 2005-12-07
698 url.removePTP();
699 if (url.contains("/")) {
700 String tpath("/");
701 tpath += url.after("/");
702 url = url.before("/");
703 tpath.hexDecode();
704 tpath.realPath();
705 url += tpath; // will resolve ../ and %2e2e/ and // etc
706 }
707 if (url.endsWith("/")) {
708 url.chop(); // chop off trailing / if any
709 }
710 // re-add the PTP
711 /*if (ptp.length() > 0)
712 url = ptp + "//" + url;*/
713 DEBUG_regexp("inRegExpURLList (processed): ", url);
714 unsigned int i = 0;
715 for (std::deque<RegExp>::iterator j = list_comp.begin(); j != list_comp.end(); j++) {
716 if (o.lm.l[list_ref[i]]->isNow()) {
717 if (j->match(url.toCharArray(), Rre))
718 return i;
719 }
720 else
721 DEBUG_regexp("Outside included regexp list's time limit");
722 i++;
723 }
724 }
725 else {
726 DEBUG_regexp("Outside top level regexp list's time limit");
727 }
728 return -1;
729 }
730
731 // Does a regexp search and replace.
732 // urlRegExp Code originally from from Ton Gorter 2004
733 bool ListMeta::regExp(String &line, std::deque<RegExp> &regexp_list, std::deque<String> &replacement_list) {
734 RegExp *re;
735 RegResult Rre;
736 String replacement;
737 String repstr;
738 String newLine;
739 bool linemodified = false;
740 unsigned int i;
741 unsigned int j, k;
742 unsigned int s = regexp_list.size();
743 unsigned int matches, submatches;
744 unsigned int match;
745 unsigned int srcoff;
746 unsigned int nextoffset;
747 unsigned int matchlen;
748 unsigned int oldlinelen;
749
750 if ((line.empty()) || line.length() < 3)
751 return false;
752
753 // iterate over our list of precompiled regexes
754 for (i = 0; i < s; i++) {
755 newLine = "";
756 re = &(regexp_list[i]);
757 if (re->match(line.toCharArray(), Rre)) {
758 repstr = replacement_list[i];
759 matches = Rre.numberOfMatches();
760
761 srcoff = 0;
762
763 for (j = 0; j < matches; j++) {
764 nextoffset = Rre.offset(j);
765 matchlen = Rre.length(j);
766
767 // copy next chunk of unmodified data
768 if (nextoffset > srcoff) {
769 newLine += line.subString(srcoff, nextoffset - srcoff);
770 srcoff = nextoffset;
771 }
772
773 // Count number of submatches (brackets) in replacement string
774 for (submatches = 0; j + submatches + 1 < matches; submatches++)
775 if (Rre.offset(j + submatches + 1) + Rre.length(j + submatches + 1) > srcoff + matchlen)
776 break;
777
778 // \1 and $1 replacement
779 replacement = "";
780 for (k = 0; k < repstr.length(); k++) {
781 // find \1..\9 and $1..$9 and fill them in with submatched strings
782 if ((repstr[k] == '\\' || repstr[k] == '$') && repstr[k + 1] >= '1' && repstr[k + 1] <= '9') {
783 match = repstr[++k] - '0';
784 if (match <= submatches) {
785 replacement += Rre.result(j + match).c_str();
786 }
787 } else {
788 // unescape \\ and \$, and add non-backreference characters to string
789 if (repstr[k] == '\\' && (repstr[k + 1] == '\\' || repstr[k + 1] == '$'))
790 k++;
791 replacement += repstr.subString(k, 1);
792 }
793 }
794
795 // copy filled in replacement string
796 newLine += replacement;
797 srcoff += matchlen;
798 j += submatches;
799 }
800 oldlinelen = line.length();
801 if (srcoff < oldlinelen) {
802 newLine += line.subString(srcoff, oldlinelen - srcoff);
803 }
804 DEBUG_regexp("Line modified! (", line, " -> ", newLine, ")" );
805 // copy newLine into line and continue with other regexes
806 line = newLine;
807 linemodified = true;
808 }
809 }
810
811 return linemodified;
812
813 }
814
815 bool ListMeta::headerRegExpReplace(ListMeta::list_info &listi, std::deque<String> &header, list_result &res) {
816 // exit immediately if list is empty
817 if (not listi.comp.size())
818 return false;
819 bool result = false;
820 for (std::deque<String>::iterator i = header.begin(); i != header.end(); i++) {
821 DEBUG_regexp("Starting header reg exp replace: ", *i);
822 bool chop = false;
823 if (i->endsWith("\r")) {
824 i->chop();
825 chop = true;
826 }
827 result |= regExp(*i, listi.comp, listi.replace);
828 if (chop)
829 i->append("\r");
830 }
831
832 for (std::deque<String>::iterator i = header.begin(); i != header.end(); i++)
833 DEBUG_regexp("Starting header reg exp replace result: ", *i);
834
835 return result;
836 }
837
838 int ListMeta::inHeaderRegExp(list_info &listi, std::deque<String> &header, list_result &res, String &lastcategory) {
839 // exit immediately if list is empty
840 if (not listi.comp.size())
841 return false;
842 int result = -1;
843 for (std::deque<String>::iterator i = header.begin(); i != header.end(); i++) {
844 DEBUG_regexp("Starting header reg exp check ", *i);
845 bool chop = false;
846 if (i->endsWith("\r")) {
847 i->chop();
848 chop = true;
849 }
850 result = inRegExpURLList(*i, listi.comp, listi.reg_list_ref, listi.list_ref, lastcategory);
851 if (chop)
852 i->append("\r");
853 if (result > -1) {
854 res.category = lastcategory;
855 //res.match = TODO add the info
856 break;
857 }
858 }
859 return result;
860 }
861