Debian Codegraph @

src/ListContainer.cpp

raw
1 // ListContainer - class for both item and phrase lists
2
3 // For all support, instructions and copyright go to:
4 // http://e2guardian.org/
5 // Released under the GPL v2, with the OpenSSL exception described in the README file.
6
7 // INCLUDES
8
9 #ifdef HAVE_CONFIG_H
10 #include "e2config.h"
11 #endif
12 #include <cstdint>
13
14 #include <syslog.h>
15 #include <algorithm>
16 #include "ListContainer.hpp"
17 #include "OptionContainer.hpp"
18 #include "RegExp.hpp"
19 #include "Logger.hpp"
20
21 #include <algorithm>
22 #include <cstdlib>
23 #include <cstdio>
24 #include <ctime>
25 #include <unistd.h>
26 #include <iostream>
27 #include <fstream>
28 #include <sys/stat.h>
29 #include <sys/time.h>
30 #include <list>
31
32
33 // GLOBALS
34
35 extern OptionContainer o;
36
37 // DEFINES
38
39 #define ROOTNODESIZE 260
40 #define MAXROOTLINKS ROOTNODESIZE - 4
41 #define GRAPHENTRYSIZE 64
42 #define MAXLINKS GRAPHENTRYSIZE - 4
43 #define ROOTOFFSET ROOTNODESIZE - GRAPHENTRYSIZE
44
45 // IMPLEMENTATION
46
47
48 // Constructor - set default values
49 ListContainer::ListContainer() {
50 }
51
52 // delete the memory block when the class is destryed
53 ListContainer::~ListContainer() {
54 reset();
55 }
56
57 // for both types of list - clear & reset all values
58 void ListContainer::reset() {
59 if (data != nullptr) free(data);
60 if (graphused && realgraphdata != nullptr)
61 free(realgraphdata);
62 // dereference this and included lists
63 // - but not if the reason we're being
64 // deleted is due to deletion (this will
65 // only happen due to list garbage
66 // collection, at which point the list
67 // ref should already be zero)
68 if (refcount > 0) {
69 --refcount;
70 // DEBUG_debug("de-reffing ", sourcefile, " due to manual list reset, refcount: ", refcount);
71 for (size_t i = 0; i < morelists.size(); ++i)
72 o.lm.deRefList(morelists[i]);
73 }
74 data = NULL;
75 realgraphdata = NULL;
76 maxchildnodes = 0;
77 graphitems = 0;
78 data_length = 0;
79 data_memory = 0;
80 items = 0;
81 isSW = false;
82 issorted = false;
83 graphused = false;
84 force_quick_search = 0;
85 /*sthour = 0;
86 stmin = 0;
87 endhour = 0;
88 endmin = 0;
89 days = "";
90 timetag = "";*/
91 category = "";
92 istimelimited = false;
93 combilist.clear();
94 slowgraph.clear();
95 list.clear();
96 lengthlist.clear();
97 weight.clear();
98 itemtype.clear();
99 timelimitindex.clear();
100 morelists.clear();
101 timelimits.clear();
102 listcategory.clear();
103 categoryindex.clear();
104 used = false;
105 parent = false;
106 bannedpfile = "";
107 exceptionpfile = "";
108 weightedpfile = "";
109 bannedpfiledate = 0;
110 exceptionpfiledate = 0;
111 weightedpfiledate = 0;
112 read_errors = false;
113 if (is_iplist) {
114 iplist.clear();
115 iprangelist.clear();
116 }
117 }
118
119 // for item lists - during a config reload, can we simply retain the already loaded list?
120 bool ListContainer::previousUseItem(const char *filename, bool startswith, int filters) {
121 String f(filename);
122 if (f == sourcefile && startswith == sourcestartswith && filters == sourcefilters) {
123 return true;
124 }
125 return false;
126 }
127
128 // for phrase lists - read in the given file, which may be an exception list
129 // inherit category and time limits from parent
130 bool ListContainer::readPhraseList(const char *filename, bool isexception, int catindex, int timeindex, bool incref,
131 int nlimit) {
132 // only increment refcount on first read, not read of included files
133 // (includes get amalgamated, unlike item lists)
134 if (incref)
135 ++refcount;
136 sourcefile = filename;
137 sourceisexception = isexception;
138 std::string linebuffer; // a string line buffer ;)
139 String temp; // a String for temporary manipulation
140 String line;
141 String lcat;
142 size_t len = 0;
143 try {
144 len = getFileLength(filename);
145 } catch (std::runtime_error &e) {
146 E2LOGGER_error("Error reading file (does it exist?) ", filename, ": ", e.what());
147 o.config_error = true;
148 return false;
149 }
150 if (len < 2) {
151 return true; // its blank - perhaps due to webmin editing
152 // just return
153 }
154 filedate = getFileDate(filename);
155 if (!increaseMemoryBy(len + 2)) { // Allocate some memory to hold file
156 E2LOGGER_error("Memory allocation failed");
157 return false;
158 };
159 std::ifstream listfile(filename, std::ios::in); // open the file for reading
160 if (!listfile.good()) {
161 E2LOGGER_error("Error opening file (does it exist?): ", filename);
162 o.config_error = true;
163 return false;
164 }
165 lcat = "";
166 bool caseinsensitive = true;
167 int line_no = 0;
168 while (!listfile.eof()) { // keep going until end of file
169 getline(listfile, linebuffer); // grab a line
170 line_no++;
171 if (linebuffer.length() != 0) { // sanity checking
172 line = linebuffer.c_str();
173 line.removeWhiteSpace();
174 // convert to lowercase - unless this is, for example,
175 // a phraselist in an odd character encoding which has
176 // been marked as not to be converted
177 if (caseinsensitive)
178 line.toLower();
179 if (line.startsWith("<"))
180 readPhraseListHelper(line, isexception, catindex, timeindex, nlimit);
181 // handle included list files
182 else if (line.startsWith(".")) {
183 temp = line.after(".include<").before(">");
184 if (temp.length() > 0) {
185 if (!readPhraseList(temp.toCharArray(), isexception, catindex, timeindex, false, nlimit)) {
186 E2LOGGER_error(" at line ", line_no, " of ", filename);
187 continue;
188 // listfile.close();
189 // return false;
190 }
191 }
192 }
193 // phrase lists can be categorised (but not time limited)
194 else if (line.startsWith("#listcategory:")) {
195 //use the original line so as to preserve case in category names
196 temp = linebuffer.c_str();
197 lcat = temp.after("\"").before("\"");
198 // this serves two purposes: returning the index of the category string
199 // if it is already in our category list, and adding it to the list (also
200 // returning index) if it is not.
201 catindex = getCategoryIndex(&lcat);
202 DEBUG_debug("List category: ", lcat, "Category list index: ", catindex);
203 }
204 // phrase lists can also be marked as not to be case-converted,
205 // to aid support for exotic character encodings
206 else if (line.startsWith("#noconvert")) {
207 DEBUG_debug("List flagged as not to be case-converted");
208 caseinsensitive = false;
209 }
210 // Read in time tags; set timeindex to the ID of the new tag
211 else if (line.startsWith("#time: ")) { // see if we have a time tag
212 TimeLimit tl;
213 if (!readTimeTag(&line, tl)) {
214 return false;
215 }
216 timelimits.push_back(tl);
217 timeindex = timelimits.size() - 1;
218 DEBUG_debug("Found time limit on phrase list. Now have ", timelimits.size(), " limits on this list (including parents).");
219 continue;
220 }
221 }
222 }
223 listfile.close();
224 return true; // sucessful read
225 }
226
227 // for phrase lists - helper function for readPhraseList
228 void ListContainer::readPhraseListHelper(String line, bool isexception, int catindex, int timeindex, int &nlimit) {
229 // read in weighting value, if there
230 // 1st check for % weighting
231 int weighting = line.after("><").before("%>").toInteger();
232 if (weighting != 0) // it is a %
233 {
234 weighting = (weighting * nlimit) / 100;
235 } else {
236 // check for normal weighting
237 weighting = line.after("><").before(">").toInteger();
238 }
239 // defaults to 0
240 int type;
241 if (weighting != 0) {
242 // this is a weighted phrase
243 type = 1;
244 line = line.before("><");
245 line += ">";
246 } else {
247 if (isexception) {
248 // this is an exception phrase
249 type = -1;
250 } else {
251 type = 0;
252 }
253 }
254
255 if (line.after(">,").length() > 2) {
256 // push type & weighting for all phrases on this line onto the combi list
257 while (line.length() > 2) {
258 line = line.after("<");
259 // combination banned, weighted, or exception
260 readPhraseListHelper2(line.before(">"), type + 11, weighting, catindex, timeindex);
261 line = line.after(">,");
262 }
263 // end of combi marker
264 readPhraseListHelper2("", type + 21, weighting, catindex, timeindex);
265 } else {
266 line = line.after("<").before(">");
267 // push type & weighting for this individual phrase onto combi list (not combination phrase)
268 readPhraseListHelper2(line, type, weighting, catindex, timeindex);
269 }
270 }
271
272 // for phrase lists - push phrase, type, weighting & category onto combi list
273 void ListContainer::readPhraseListHelper2(String phrase, int type, int weighting, int catindex, int timeindex) {
274 // -1=exception
275 // 0=banned
276 // 1=weighted
277 // 10 = combination exception
278 // 11 = combination banned
279 // 12 = combination weighted
280 // 20,21,22 = end of combi marker
281
282 if (type > 19) {
283 combilist.push_back(-2); // mark an end of a combi
284 combilist.push_back(type - 21); // store the combi type
285 combilist.push_back(timeindex); // store the combi timtime limitt
286 combilist.push_back(weighting); // store the combi weight
287 combilist.push_back(catindex); // store the combi category
288 return;
289 }
290
291 phrase.removePunctuation();
292
293 if (phrase.length() > 127) {
294 E2LOGGER_error("Phrase length too long, truncating: ", phrase);
295 phrase = phrase.subString(0, 127);
296 }
297
298 if (phrase.length() < 1) { // its too small to use
299 return;
300 }
301
302 if (type < 10) {
303 if (!addToItemListPhrase(phrase.toCharArray(), phrase.length(), type, weighting, false, catindex, timeindex)) {
304 E2LOGGER_error("Duplicate phrase, dropping: ", phrase );
305 }
306 return;
307 }
308 // must be a combi or end marker if got here
309
310 // must be a combi if got here
311
312 addToItemListPhrase(phrase.toCharArray(), phrase.length(), type, weighting, true, catindex, timeindex);
313 }
314
315 // for item lists - add phrases to list proper
316 bool ListContainer::addToItemListPhrase(const char *s, size_t len, int type, int weighting, bool combi, int catindex,
317 int timeindex) {
318 list.push_back(data_length);
319 lengthlist.push_back(len);
320 for (size_t i = 0; i < len; i++) {
321 data[data_length + i] = s[i];
322 }
323 data[data_length + len] = 0;
324 data_length += len + 1;
325 if (combi) {
326 // if this is a combination item, store the ID of the current item on the combi list
327 combilist.push_back(items);
328 }
329 items++;
330 weight.push_back(weighting);
331 itemtype.push_back(type);
332 categoryindex.push_back(catindex);
333 timelimitindex.push_back(timeindex);
334 return true;
335 }
336
337 //bool ListContainer::ifsreadItemList(std::istream *input, const char *list_pwd, int len, bool checkendstring, const char *endstring, bool do_includes, bool startswith, int filters)
338 bool
339 ListContainer::ifsreadItemList(const char *filename, std::istream *input, String basedir, const char *list_pwd, int len, bool checkendstring,
340 const char *endstring, bool do_includes, bool startswith, int filters) {
341 unsigned int mem_used = 2;
342 RegExp re;
343 re.comp("^.*\\:[0-9]+\\/.*");
344 RegResult Rre;
345 if (is_iplist) {
346 #ifdef HAVE_PCRE
347 matchIP.comp("^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$");
348 matchSubnet.comp("^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$");
349 matchCIDR.comp("^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}/\\d{1,2}$");
350 matchRange.comp("^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}-\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$");
351 #else
352 matchIP.comp("^[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}$");
353 matchSubnet.comp("^[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}/[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}$");
354 matchCIDR.comp("^[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}/[0-9]{1,2}$");
355 matchRange.comp("^[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}-[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}$");
356 #endif
357 }
358
359 if (filters != 32)
360 DEBUG_config("Converting to lowercase");
361
362 if (!increaseMemoryBy(len + 2)) {
363 E2LOGGER_error("Memory allocation failed");
364 return false;
365 }; // Allocate some memory to hold file
366 String temp, inc, hostname, url;
367 //char linebuffer[2048];
368 char linebuffer[20000]; // increased to allow checking of line length
369
370
371 while (!input->eof()) {
372 input->getline(linebuffer, sizeof(linebuffer));
373 temp = linebuffer;
374 if (temp.length() < 2)
375 continue; // its jibberish
376 if (temp.length() > 2048) {
377 temp.limitLength(100);
378 E2LOGGER_error("Line too long in list file - ignored ", temp );
379 continue;
380 }
381
382 //item lists (URLs, domains) can be both categorised and time-limited
383 if (linebuffer[0] == '#') {
384 if (temp.startsWith("#time: ")) { // see if we have a time tag
385 if (!readTimeTag(&temp, listtimelimit)) {
386 return false;
387 }
388 continue;
389 } else if (temp.startsWith("#listcategory:")) {
390 category = temp.after("\"").before("\"");
391 DEBUG_config("found item list category: ", category);
392 continue;
393 } else if (checkendstring && temp.startsWith(endstring)) {
394 break;
395 }
396 continue; // it's a comment
397 }
398
399
400 // Strip off comments that don't necessarily start at the beginning of a line
401 // - but not regular expression comments
402 // - or '#' within a URL
403 // So only strings starting with ' #' are regarded as comments
404 // if the '#' is not at start of line
405 //
406 // Amended by Philip Pearce - included in e2g first release 2013:
407 std::string::size_type commentstart = 1;
408 while ((commentstart = temp.find_first_of('#', commentstart)) != std::string::npos) {
409 // Don't treat "(?#...)" as a E2 comment - it's a regex comment
410 // if (temp[commentstart - 1] != '?')
411 // Changed to only treat ' #' as an embeded comment
412 if (temp[commentstart - 1] == ' ') {
413 temp = temp.substr(0, commentstart);
414 break;
415 }
416 ++commentstart;
417 }
418
419 temp.removeWhiteSpace(); // tidy up and make it handle CRLF files
420 if (temp.startsWith(".Include<")) { // see if we have another list
421 if (do_includes) {
422 inc = temp.after(".Include<"); // to include
423 inc = inc.before(">");
424 if (inc.contains("__LISTDIR__")) {
425 String inc2 = inc.before("__LISTDIR__");
426 inc2 += list_pwd;
427 inc2 += inc.after("__LISTDIR__");
428 inc = inc2;
429 }
430 inc.fullPath(basedir);
431 int list_no = -1;
432 if (!readAnotherItemList(inc.toCharArray(), list_pwd, startswith, filters, list_no)) { // read it
433 read_errors = true;
434 // return false;
435 } else {
436 if ( list_no > -1 && o.lm.l[list_no]->read_errors ) {
437 read_errors = true;
438 }
439 }
440 }
441 continue;
442 }
443 if (!(is_iplist || is_timelist)) {
444 if (filters != 32) {
445 if (temp.endsWith("/")) {
446 temp.chop(); // tidy up
447 }
448 if (temp.startsWith("ftp://")) {
449 temp = temp.after("ftp://"); // tidy up
450 }
451 }
452 if (filters == 1) { // remove port addresses
453 if (temp.before("/").contains(":")) { // quicker than full regexp
454 if (re.match(temp.toCharArray(), Rre)) {
455 hostname = temp.before(":");
456 url = temp.after("/");
457 temp = hostname;
458 temp += "/";
459 temp += url;
460 }
461 }
462 }
463 if (filters != 32) {
464 temp.toLower(); // tidy up - but don't make regex lists lowercase!
465 }
466 }
467 if (temp.length() > 0) {
468 mem_used += temp.length() + 1;
469 if (is_iplist) {
470 addToIPList(temp);
471 } else if (is_timelist) {
472 if (!addToTimeList(temp))
473 return false;
474 } else if (is_map) {
475 addToDataMap(temp);
476 } else {
477 if (mem_used > data_memory) {
478 if(!increaseMemoryBy(20000)) {
479 E2LOGGER_error("Memory allocation error");
480 return false;
481 };
482 }
483 addToItemList(temp.toCharArray(), temp.length()); // add to unsorted list
484 }
485 }
486 }
487
488 if (is_iplist) {
489 if (is_map) {
490 std::stable_sort(ipmaplist.begin(), ipmaplist.end());
491 std::stable_sort(ipmaprangelist.begin(), ipmaprangelist.end());
492 issorted = true;
493 // temp code for testing
494 if (false) {
495 std::cerr << "ipmaplist size is " << ipmaplist.size() << std::endl;
496 std::vector<String> iplist2{
497 "10.81.64.5",
498 "10.81.76.23",
499 "10.81.65.12",
500 "10.81.65.16",
501 "10.81.65.22",
502 "10.81.65.25",
503 "10.81.65.29",
504 "10.81.65.31",
505 "10.81.65.33",
506 "10.81.65.35",
507 "10.81.65.36",
508 "10.81.65.37",
509 "10.81.65.39",
510 "10.81.65.40",
511 "10.81.65.41",
512 "10.81.65.45",
513 "10.81.65.48",
514 "10.81.65.49",
515 "10.81.65.50",
516 "10.81.65.51",
517 "10.81.65.52",
518 "10.81.65.55",
519 "10.81.65.56",
520 "10.81.65.57",
521 "10.81.65.58",
522 "10.81.65.59",
523 "10.81.65.63",
524 "10.81.65.60",
525 "10.81.65.93",
526 "10.81.65.159",
527 "10.81.66.13",
528 "10.81.66.18",
529 "10.81.66.20",
530 "10.81.66.25",
531 "10.81.66.26",
532 "10.81.66.28",
533 "10.81.66.34",
534 "10.81.66.36",
535 "10.81.66.37",
536 "10.81.66.39",
537 "10.81.66.40",
538 "10.81.66.42",
539 "10.81.66.43",
540 "10.81.66.44",
541 "10.81.66.47",
542 "10.81.66.51",
543 "10.81.66.58",
544 "10.81.66.64",
545 "10.81.66.75",
546 "10.81.67.10",
547 "10.81.67.11",
548 "10.81.67.12",
549 "10.81.69.10",
550 "10.81.70.13",
551 "10.81.70.14",
552 "192.168.206.27",
553 "192.168.206.30",
554 "192.168.206.35",
555 "192.168.206.36",
556 };
557 for (auto item: iplist2) {
558 String res = getIPMapData(item);
559 if (res.empty()) {
560 std::cerr << "IP " << item << " NOT found" << std::endl;
561 } else {
562 std::cerr << "IP " << item << " found group " << res << std::endl;
563 }
564
565 }
566 }
567 } else {
568 std::stable_sort(iplist.begin(), iplist.end());
569 std::stable_sort(iprangelist.begin(), iprangelist.end());
570 issorted = true;
571
572 }
573 } else if (is_map) {
574 std::stable_sort(datamaplist.begin(), datamaplist.end());
575 issorted = true;
576 if(false) { // make true for testing
577 String match, result;
578 for (auto item : datamaplist) {
579 std::cerr << item.key << " mapped to " << item.group << std::endl;
580 }
581 std::cerr << "End of list" << std::endl;
582 String t = "philip";
583 String tg = getMapData(t, match, result);
584 std::cerr << "Search for philip got " << tg.c_str() << std::endl;
585 }
586 }
587 if ((mem_used + 20) < data_memory) { // free up unused memory
588 data = (char *) realloc(data, (mem_used + 20) * sizeof(char));
589 if (data == nullptr)
590 return false;
591 }
592
593 if(read_errors) {
594 E2LOGGER_warning("**List Error** There are read failures in some lists included in ", filename);
595 }
596
597 return true; // sucessful read
598 }
599
600 bool ListContainer::ifsReadSortItemList(std::ifstream *input, String basedir, const char *list_pwd, bool checkendstring,
601 const char *endstring, bool do_includes, bool startswith, int filters,
602 const char *filename) {
603 size_t len = 0;
604 try {
605 len = getFileLength(filename);
606 } catch (std::runtime_error &e) {
607 E2LOGGER_error("Error reading file ", filename, ": ", e.what());
608 return false;
609 }
610 bool ret;
611 ret = ifsreadItemList(filename,input, basedir, list_pwd, len, checkendstring, endstring, do_includes, startswith, filters);
612 if (ret) {
613 doSort(startswith);
614 return true;
615 }
616 return false;
617 }
618
619 // for item lists - read item list from file. checkme - what is startswith? is it used? what is filters?
620 bool ListContainer::readItemList(const char *filename, const char *list_pwd, bool startswith, int filters, bool isip,
621 bool istime, bool ismap) {
622 ++refcount;
623 sourcefile = filename;
624 sourcestartswith = startswith;
625 sourcefilters = filters;
626
627 if (isip) is_iplist = true;
628 else is_iplist = false;
629
630 if (istime) is_timelist = true;
631 else is_timelist = false;
632
633 if (ismap) is_map = true;
634 else is_map = false;
635
636 if (sourcefile.startsWithLower("memory:"))
637 return readStdinItemList(startswith, filters, filename);
638 std::string linebuffer;
639 DEBUG_config(filename);
640
641 //struct stat s;
642 filedate = getFileDate(filename);
643 size_t len = 0;
644 if (filedate == 0) return false;
645 try {
646 len = getFileLength(filename);
647 } catch (std::runtime_error &e) {
648 E2LOGGER_error("*List Error* opening file ", filename, ": ", e.what());
649 return false;
650 }
651 if (len < 2) {
652 return true; // its blank - perhaps due to webmin editing
653 // just return
654 }
655 std::ifstream listfile(filename, std::ios::in);
656 if (!listfile.good()) {
657 E2LOGGER_error("*List Error* opening: ", filename);
658 return false;
659 }
660 String base_dir(filename);
661 base_dir.baseDir();
662 if (!ifsreadItemList(filename,&listfile, base_dir, list_pwd, len, false, NULL, true, startswith, filters)) {
663 listfile.close();
664 E2LOGGER_error("*List Error* while reading: ", filename);
665 return false;
666 }
667 listfile.close();
668 return true; // sucessful read
669 }
670
671 // for stdin item lists - read item list from stdin
672 bool ListContainer::readStdinItemList(bool startswith, int filters, const char *filename) {
673 if (filters != 32)
674 DEBUG_config("Converting to lowercase");
675 std::string linebuffer;
676 RegExp re;
677 re.comp("^.*\\:[0-9]+\\/.*");
678 RegResult Rre;
679 size_t len = 2046;
680 if (!increaseMemoryBy(20000)) { // Allocate some memory to hold list
681 E2LOGGER_error("Memory allocation failed");
682 return false;
683 };
684 if (!std::cin.good()) {
685 E2LOGGER_error("Error reading stdin: ");
686 return false;
687 }
688
689 if (!ifsreadItemList(filename, &std::cin, "", "", len, true, "#ENDLIST", false, startswith, filters)) {
690 E2LOGGER_error("Error reading stdin: ");
691 return false;
692 } else
693 return true;
694 }
695
696
697 // for item lists - read nested item lists
698 bool ListContainer::readAnotherItemList(const char *filename, const char *list_pwd, bool startswith, int filters, int &result) {
699 result = o.lm.newItemList(filename, list_pwd, startswith, filters, false, is_iplist, is_timelist, is_map);
700 if (result < 0) {
701 //E2LOGGER_error("*List Error* unable to read file: ", filename);
702 return false;
703 }
704 morelists.push_back((unsigned) result);
705 return true;
706 }
707
708 // for item lists - is this item in the list?
709 bool ListContainer::inList(const char *string, String &lastcategory) {
710 String match, result;
711 if (findInList(string, lastcategory, match, result)) {
712 return true;
713 }
714 return false;
715 }
716
717 // for item lists - is an item in the list that ends with this string?
718 bool ListContainer::inListEndsWith(const char *string, String &lastcategory) {
719 if (isNow()) {
720 if (items > 0) {
721 if (search(&ListContainer::greaterThanEW, 0, items - 1, string) >= 0) {
722 lastcategory = category;
723 return true;
724 }
725 }
726 bool rc;
727 for (unsigned int i = 0; i < morelists.size(); i++) {
728 rc = (*o.lm.l[morelists[i]]).inListEndsWith(string, lastcategory);
729 if (rc) {
730 // lastcategory = (*o.lm.l[morelists[i]]).lastcategory;
731 return true;
732 }
733 }
734 }
735 return false;
736 }
737
738 // for item lists - is an item in the list that starts with this string?
739 bool ListContainer::inListStartsWith(const char *string, String &lastcategory) {
740 if (isNow()) {
741 if (items > 0) {
742 if (search(&ListContainer::greaterThanSW, 0, items - 1, string) >= 0) {
743 lastcategory = category;
744 return true;
745 }
746 }
747 bool rc;
748 for (unsigned int i = 0; i < morelists.size(); i++) {
749 rc = (*o.lm.l[morelists[i]]).inListStartsWith(string, lastcategory);
750 if (rc) {
751 //lastcategory = (*o.lm.l[morelists[i]]).lastcategory;
752 return true;
753 }
754 }
755 }
756 return false;
757 }
758
759 // find pointer to the part of the data array containing this string
760 bool ListContainer::findInList(const char *string, String &lastcategory, String &match, String &result) {
761 if (isNow()) {
762 if (is_iplist) {
763 if (is_map) {
764 std::string sstring = string;
765 String rcs = getIPMapData(sstring);
766 if (rcs != "") {
767 result = rcs;
768 return true;
769 }
770 } else if (inIPList(string, match)) {
771 lastcategory = category;
772 return true;
773 }
774 } else if (is_map) {
775 String sstring;
776 sstring = string;
777 auto rcs = getMapData(sstring, match, result);
778 if (rcs) {
779 return true;
780 }
781 } else if (items > 0) {
782 int r;
783 if (isSW) {
784 r = search(&ListContainer::greaterThanSWF, 0, items - 1, string);
785 } else {
786 r = search(&ListContainer::greaterThanEWF, 0, items - 1, string);
787 }
788 if (r >= 0) {
789 lastcategory = category;
790 match = (data + list[r]);
791 return true;
792 }
793 }
794 bool rc;
795 for (unsigned int i = 0; i < morelists.size(); i++) {
796 rc = (*o.lm.l[morelists[i]]).findInList(string, lastcategory, match, result);
797 if (rc) {
798 //lastcategory = (*o.lm.l[morelists[i]]).lastcategory;
799 return rc;
800 }
801 }
802 }
803 return false;
804 }
805
806 // find an item in the list which starts with this
807 bool ListContainer::findStartsWith(const char *string, String &lastcategory, String &match) {
808 if (isNow()) {
809 if (items > 0) {
810 int r = search(&ListContainer::greaterThanSW, 0, items - 1, string);
811 if (r >= 0) {
812 lastcategory = category;
813 match = (data + list[r]);
814 return true;
815 }
816 }
817 bool rc;
818 for (unsigned int i = 0; i < morelists.size(); i++) {
819 rc = (*o.lm.l[morelists[i]]).findStartsWith(string, lastcategory, match);
820 if (rc) {
821 return rc;
822 }
823 }
824 }
825 return false;
826 }
827
828
829 bool ListContainer::findEndsWith(const char *string, String &lastcategory, String &match) {
830 if (isNow()) {
831 if (items > 0) {
832 int r = search(&ListContainer::greaterThanEW, 0, items - 1, string);
833 if (r >= 0) {
834 lastcategory = category;
835 match = (data + list[r]);
836 return true;
837 }
838 }
839 bool rc;
840 for (unsigned int i = 0; i < morelists.size(); i++) {
841 rc = (*o.lm.l[morelists[i]]).findEndsWith(string, lastcategory, match);
842 if (rc) {
843 return rc;
844 }
845 }
846 }
847 return false;
848 }
849
850 // For phrase lists - grab the text, score and type of a given phrase, based on item number within list
851 std::string ListContainer::getItemAtInt(int index) {
852 std::string s(data + list[index], lengthlist[index]);
853 return s;
854 }
855
856 int ListContainer::getWeightAt(unsigned int index) {
857 return weight[index];
858 }
859
860 int ListContainer::getTypeAt(unsigned int index) {
861 return itemtype[index];
862 }
863 // Phrase lists - check whether the current time is within the limit imposed upon the given phrase
864 bool ListContainer::checkTimeAt(unsigned int index) {
865 if (timelimitindex[index] == -1) {
866 return true;
867 }
868 return isNow(timelimitindex[index]);
869 }
870
871 bool ListContainer::checkTimeAtD(int index) {
872 if (index == -1) {
873 return true;
874 }
875 return isNow(index);
876 }
877
878 // The following comparator function will result in sort in descending order - which is counter-intuitive!!!
879 // Sorts backwards within string - used for site lists which are more significant to the right
880 struct lessThanEWF : public std::binary_function<const size_t &, const size_t &, bool> {
881 bool operator()(const size_t &aoff, const size_t &boff) {
882 const char *a = data + aoff;
883 const char *b = data + boff;
884 size_t alen = strlen(a);
885 size_t blen = strlen(b);
886 size_t apos = alen - 1;
887 size_t bpos = blen - 1;
888 for (size_t maxlen = ((alen < blen) ? alen : blen); maxlen > 0; apos--, bpos--, maxlen--) {
889 if (a[apos] > b[bpos])
890 return true;
891 else if (a[apos] < b[bpos])
892 return false;
893 }
894 if (alen >= blen)
895 return true;
896 else //if (alen < blen)
897 return false;
898 //return true; // both equal
899 };
900 char *data;
901 };
902
903 // The following comparator function will result in sort in descending order - which is counter-intuitive!!!
904 // Sorts forwards within string - used for url and other lists which are more significant to the left
905 struct lessThanSWF : public std::binary_function<const size_t &, const size_t &, bool> {
906 bool operator()(const size_t &aoff, const size_t &boff) {
907 const char *a = data + aoff;
908 const char *b = data + boff;
909 size_t alen = strlen(a);
910 size_t blen = strlen(b);
911 size_t maxlen = (alen < blen) ? alen : blen;
912 for (size_t i = 0; i < maxlen; i++) {
913 if (a[i] > b[i])
914 return true;
915 else if (a[i] < b[i])
916 return false;
917 }
918 if (alen >= blen)
919 return true;
920 else //if (alen > blen)
921 return false;
922 };
923 char *data;
924 };
925
926 void ListContainer::doSort(const bool startsWith) { // sort by ending of line
927 for (size_t i = 0; i < morelists.size(); i++)
928 (*o.lm.l[morelists[i]]).doSort(startsWith);
929 if (is_iplist) {
930 if (is_map)
931 std::stable_sort(ipmaplist.begin(), ipmaplist.end());
932 else
933 std::stable_sort(iplist.begin(), iplist.end());
934
935 return;
936 }
937 if (is_map) { // deal with datamaplist
938 std::stable_sort(datamaplist.begin(), datamaplist.end());
939 return;
940 }
941
942 if (items < 2 || issorted)
943 return;
944 if (startsWith) {
945 lessThanSWF lts;
946 lts.data = data;
947 std::stable_sort(list.begin(), list.end(), lts);
948 } else {
949 lessThanEWF lte;
950 lte.data = data;
951 std::stable_sort(list.begin(), list.end(), lte);
952 }
953 isSW = startsWith;
954 issorted = true;
955 #ifdef DEBUG_LOW
956 self_check();
957 #endif
958 return;
959 }
960
961
962 bool ListContainer::makeGraph(bool fqs) {
963 force_quick_search = fqs;
964 if (data_length == 0)
965 return true;
966 long int i;
967 // Quick search has been forced on - put all items on the "slow" list and be done with it
968 if (force_quick_search) {
969 for (i = 0; i < items; i++) {
970 // Check to see if the item is a duplicate
971 std::string thisphrase = getItemAtInt(i);
972 bool found = false;
973 unsigned int foundindex = 0;
974 for (std::vector<unsigned int>::iterator j = slowgraph.begin(); j != slowgraph.end(); j++) {
975 if (getItemAtInt(*j) == thisphrase) {
976 found = true;
977 foundindex = *j;
978 break;
979 }
980 }
981 if (!found) {
982 // Not a duplicate - store it
983 slowgraph.push_back(i);
984 } else {
985 // Duplicate - resolve the collision
986 //
987 // Existing entry must be a combi AND
988 // new entry is not a combi so we overwrite the
989 // existing values as combi values and types are
990 // stored in the combilist
991 // OR
992 // both are weighted phrases and the new phrase is higher weighted
993 // OR
994 // the existing phrase is weighted and the new phrase is banned
995 // OR
996 // new phrase is an exception; exception phrases take precedence
997 if ((itemtype[foundindex] > 9 && itemtype[i] < 10) ||
998 (itemtype[foundindex] == 1 && itemtype[i] == 1 && (weight[i] > weight[foundindex])) ||
999 (itemtype[foundindex] == 1 && itemtype[i] == 0) || itemtype[i] == -1) {
1000 itemtype[foundindex] = itemtype[i];
1001 weight[foundindex] = weight[i];
1002 categoryindex[foundindex] = categoryindex[i];
1003 timelimitindex[foundindex] = timelimitindex[i];
1004 }
1005 }
1006 }
1007 return true;
1008 }
1009 std::string s;
1010 std::string lasts;
1011 graphused = true;
1012
1013 #ifdef DEBUG_LOW
1014 DEBUG_debug("Bytes needed for phrase tree in worst-case scenario: ", (sizeof(int) * ((GRAPHENTRYSIZE * data_length) + ROOTOFFSET)),
1015 ", starting off with allocation of ", (sizeof(int) * ((GRAPHENTRYSIZE * ((data_length / 3) + 1)) + ROOTOFFSET)) );
1016 prolificroot = false;
1017 secondmaxchildnodes = 0;
1018 #endif
1019
1020 // Make a conservative guess at how much memory will be needed - call realloc() as necessary to change what is actually taken
1021 current_graphdata_size = (GRAPHENTRYSIZE * ((data_length / 3) + 1)) + ROOTOFFSET;
1022 realgraphdata = (int *) calloc(current_graphdata_size, sizeof(int));
1023 if (realgraphdata == NULL) {
1024 E2LOGGER_error("Cannot allocate memory for phrase tree: ", strerror(errno));
1025 return false;
1026 }
1027 graphitems++;
1028 std::deque<size_t> sizelist;
1029
1030 for (i = 0; i < items; i++) {
1031 sizelist.push_back(i);
1032 }
1033 graphSizeSort(0, items - 1, &sizelist);
1034
1035 for (i = 0; i < items; i++) {
1036 graphAdd(String(data + list[sizelist[i]], lengthlist[sizelist[i]]), 0, sizelist[i]);
1037 }
1038
1039 DEBUG_config("Bytes actually needed for phrase tree: ", (sizeof(int) * ((GRAPHENTRYSIZE * graphitems) + ROOTOFFSET)) );
1040 DEBUG_config("Most prolific node has ", maxchildnodes, " children");
1041 DEBUG_config("It ", (prolificroot ? "is" : "is not"), " the root node");
1042 DEBUG_config("Second most prolific node has ", secondmaxchildnodes, " children");
1043
1044 realgraphdata = (int *) realloc(realgraphdata, sizeof(int) * ((GRAPHENTRYSIZE * graphitems) + ROOTOFFSET));
1045 if (realgraphdata == NULL) {
1046 E2LOGGER_error("Cannot reallocate memory for phrase tree: ", strerror(errno));
1047 return false;
1048 }
1049
1050 int ml = realgraphdata[2];
1051 int branches;
1052
1053 for (i = ml - 1; i >= 0; i--) {
1054 branches = graphFindBranches(realgraphdata[4 + i]);
1055 if (branches < 12) { // quicker to use B-M on node with few branches
1056 graphCopyNodePhrases(realgraphdata[4 + i]);
1057 // remove link to this node and so effectively remove all nodes
1058 // it links to but don't recover the memory as its not worth it
1059 for (int j = i; j < ml; j++) {
1060 realgraphdata[4 + j] = realgraphdata[4 + j + 1];
1061 }
1062 realgraphdata[2]--;
1063 }
1064 }
1065 return true;
1066 }
1067
1068 void ListContainer::graphSizeSort(int l, int r, std::deque<size_t> *sizelist) {
1069 if (r <= l)
1070 return;
1071 size_t e;
1072 int k;
1073 size_t v = getItemAtInt((*sizelist)[r]).length();
1074 int i = l - 1, j = r, p = i, q = r;
1075 for (;;) {
1076 while (getItemAtInt((*sizelist)[++i]).length() < v);
1077 while (v < getItemAtInt((*sizelist)[--j]).length()) {
1078 if (j == l)
1079 break;
1080 }
1081 if (i >= j)
1082 break;
1083 e = (*sizelist)[i];
1084 (*sizelist)[i] = (*sizelist)[j];
1085 (*sizelist)[j] = e;
1086 if (v == getItemAtInt((*sizelist)[i]).length()) {
1087 p++;
1088 e = (*sizelist)[p];
1089 (*sizelist)[p] = (*sizelist)[i];
1090 (*sizelist)[i] = e;
1091 }
1092 if (v == getItemAtInt((*sizelist)[j]).length()) {
1093 q--;
1094 e = (*sizelist)[q];
1095 (*sizelist)[q] = (*sizelist)[j];
1096 (*sizelist)[j] = e;
1097 }
1098 }
1099 e = (*sizelist)[i];
1100 (*sizelist)[i] = (*sizelist)[r];
1101 (*sizelist)[r] = e;
1102 j = i - 1;
1103 i++;
1104 for (k = l; k <= p; k++, j--) {
1105 e = (*sizelist)[k];
1106 (*sizelist)[k] = (*sizelist)[j];
1107 (*sizelist)[j] = e;
1108 }
1109 for (k = r - 1; k >= q; k--, i++) {
1110 e = (*sizelist)[k];
1111 (*sizelist)[k] = (*sizelist)[i];
1112 (*sizelist)[i] = e;
1113 }
1114 graphSizeSort(l, j, sizelist);
1115 graphSizeSort(i, r, sizelist);
1116 }
1117
1118 // find the total number of children a node has, along all branches
1119 int ListContainer::graphFindBranches(unsigned int pos) {
1120 int branches = 0;
1121 int *graphdata;
1122 if (pos == 0)
1123 graphdata = realgraphdata;
1124 else
1125 graphdata = realgraphdata + ROOTOFFSET;
1126 int links = graphdata[pos * GRAPHENTRYSIZE + 2];
1127 for (int i = 0; i < links; i++) {
1128 branches += graphFindBranches(graphdata[pos * GRAPHENTRYSIZE + 4 + i]);
1129 }
1130 if (links > 1) {
1131 branches += links - 1;
1132 }
1133
1134 return branches;
1135 }
1136
1137 // copy all phrases starting from a given root link into the slowgraph
1138 void ListContainer::graphCopyNodePhrases(unsigned int pos) {
1139 int *graphdata;
1140 if (pos == 0)
1141 graphdata = realgraphdata;
1142 else
1143 graphdata = realgraphdata + ROOTOFFSET;
1144 int links = graphdata[pos * GRAPHENTRYSIZE + 2];
1145 int i;
1146 for (i = 0; i < links; i++) {
1147 graphCopyNodePhrases(graphdata[pos * GRAPHENTRYSIZE + 4 + i]);
1148 }
1149 bool found = false;
1150 unsigned int foundindex = 0;
1151 unsigned int phrasenumber = graphdata[pos * GRAPHENTRYSIZE + 3];
1152 std::string thisphrase = getItemAtInt(phrasenumber);
1153 for (std::vector<unsigned int>::iterator i = slowgraph.begin(); i != slowgraph.end(); i++) {
1154 if (getItemAtInt(*i) == thisphrase) {
1155 found = true;
1156 foundindex = *i;
1157 break;
1158 }
1159 }
1160 if (!found) {
1161 slowgraph.push_back(phrasenumber);
1162 } else {
1163 // Duplicate - resolve the collision
1164 //
1165 // Existing entry must be a combi AND
1166 // new entry is not a combi so we overwrite the
1167 // existing values as combi values and types are
1168 // stored in the combilist
1169 // OR
1170 // both are weighted phrases and the new phrase is higher weighted
1171 // OR
1172 // the existing phrase is weighted and the new phrase is banned
1173 // OR
1174 // new phrase is an exception; exception phrases take precedence
1175 if ((itemtype[foundindex] > 9 && itemtype[phrasenumber] < 10) ||
1176 (itemtype[foundindex] == 1 && itemtype[phrasenumber] == 1 && (weight[phrasenumber] > weight[foundindex])) ||
1177 (itemtype[foundindex] == 1 && itemtype[phrasenumber] == 0) || itemtype[phrasenumber] == -1) {
1178 itemtype[foundindex] = itemtype[phrasenumber];
1179 weight[foundindex] = weight[phrasenumber];
1180 categoryindex[foundindex] = categoryindex[phrasenumber];
1181 timelimitindex[foundindex] = timelimitindex[phrasenumber];
1182 }
1183 }
1184 }
1185
1186 int ListContainer::bmsearch(char *file, off_t fl, const std::string &s) {
1187 off_t pl = s.length();
1188 if (fl < pl)
1189 return 0; // reality checking
1190 if (pl > 126)
1191 return 0; // reality checking
1192
1193 // must match all
1194 off_t j, l; // counters
1195 int p; // to hold precalcuated value for speed
1196 bool match; // flag
1197 int qsBc[256]; // Quick Search Boyer Moore shift table (256 alphabet)
1198 char *k; // pointer used in matching
1199
1200 int count = 0;
1201
1202 char *phrase = new char[pl + 1];
1203 for (j = 0; j < pl; j++) {
1204 phrase[j] = s[j];
1205 }
1206 phrase[pl] = 0;
1207
1208 // For speed we append the phrase to the end of the memory block so it
1209 // is always found, thus eliminating some checking. This is possible as
1210 // we know an extra 127 bytes have been provided by NaughtyFilter.cpp
1211 // and also the OptionContainer does not allow phrase lengths greater
1212 // than 126 chars
1213 k = file + fl;
1214 for (j = 0; j < pl; j++) {
1215 k[j] = s[j];
1216 }
1217
1218 // Next we need to make the Quick Search Boyer Moore shift table
1219
1220 p = pl + 1;
1221 for (j = 0; j < 256; j++) { // Preprocessing
1222 qsBc[j] = p;
1223 }
1224 for (j = 0; j < pl; j++) { // Preprocessing
1225 qsBc[(unsigned char) phrase[j]] = pl - j;
1226 }
1227
1228 // Now do the searching!
1229
1230 for (j = 0;;) {
1231 k = file + j;
1232 match = true;
1233 for (l = 0; l < pl; l++) { // quiv, but faster, memcmp()
1234 if (k[l] != phrase[l]) {
1235 match = false;
1236 break;
1237 }
1238 }
1239 if (match) {
1240 if (j >= fl) {
1241 break; // is the end of file marker
1242 }
1243 count++;
1244 }
1245 j += qsBc[(unsigned char) file[j + pl]]; // shift
1246 }
1247 delete[] phrase;
1248 return count;
1249 }
1250
1251 // Format of the data is each entry has GRAPHENTRYSIZE int values with format of:
1252 // [letter][last letter flag][num links][from phrase][link0][link1]...
1253
1254 void ListContainer::graphSearch(std::map<std::string, std::pair<unsigned int, int> > &result, char *doc, off_t len) {
1255 off_t i, j, k;
1256 std::map<std::string, std::pair<unsigned int, int> >::iterator existingitem;
1257
1258 //do standard quick search on short branches (or everything, if force_quick_search is on)
1259 for (std::vector<unsigned int>::iterator i = slowgraph.begin(); i != slowgraph.end(); i++) {
1260 std::string phrase = getItemAtInt(*i);
1261 j = bmsearch(doc, len, phrase);
1262 for (k = 0; k < j; k++) {
1263 existingitem = result.find(phrase);
1264 if (existingitem == result.end()) {
1265 result[phrase] = std::pair<unsigned int, int>(*i, 1);
1266 } else {
1267 existingitem->second.second++;
1268 }
1269 }
1270 }
1271
1272 if (force_quick_search || graphitems == 0) {
1273 #ifdef DEBUG_LOW
1274 DEBUG_debug("Map (quicksearch) start");
1275 for (std::map<std::string, std::pair<unsigned int, int> >::iterator i = result.begin(); i != result.end(); i++) {
1276 DEBUG_debug("Map: ", i->first, " ", i->second.second);
1277 }
1278 DEBUG_debug("Map (quicksearch) end");
1279 #endif
1280 return;
1281 }
1282
1283 off_t sl;
1284 off_t ppos;
1285 off_t currnode = 0;
1286 int *graphdata = realgraphdata;
1287 off_t ml;
1288 char p;
1289 off_t pos;
1290 off_t depth;
1291 // number of links from root node to first letter of phrase
1292 ml = graphdata[2] + 4;
1293 // iterate over entire document
1294 for (i = 0; i < len; i++) {
1295 // iterate over all children of the root node
1296 for (j = 4; j < ml; j++) {
1297 // grab the endpoint of this link
1298 pos = realgraphdata[j];
1299 sl = 0;
1300
1301 // now comes the main graph search!
1302 // this is basically a depth-first tree search
1303 depth = 0;
1304 while (true) {
1305 // get the address of the link endpoint and the data actually stored at it
1306 // note that this only works for GRAPHENTRYSIZE == 64
1307 ppos = pos << 6;
1308 if (ppos == 0)
1309 graphdata = realgraphdata;
1310 else
1311 graphdata = realgraphdata + ROOTOFFSET;
1312 p = graphdata[ppos];
1313
1314 // does the character at this string depth match the relevant character in the node we're currently looking at?
1315 if (p == doc[i + depth]) {
1316 // it does!
1317 // is this graph node marked as being the end of a phrase?
1318 if (graphdata[ppos + 1] == 1) {
1319 // it is, so store the pointer to the matched phrase.
1320 std::string phrase = getItemAtInt(graphdata[ppos + 3]);
1321 existingitem = result.find(phrase);
1322 if (existingitem == result.end()) {
1323 result[phrase] = std::pair<unsigned int, int>(graphdata[ppos + 3], 1);
1324 } else {
1325 existingitem->second.second++;
1326 }
1327 DEBUG_debug("Found this phrase: ", phrase);
1328 }
1329 // grab this node's number of children
1330 sl = graphdata[ppos + 2];
1331 if (sl > 0) {
1332 // this is now the node we're interested in looking at the children of
1333 currnode = ppos;
1334 // zip straight to the first child of the matched node
1335 // (this is the magic that makes it depth first)
1336 pos = graphdata[ppos + 4];
1337 depth++;
1338 continue;
1339 }
1340 // if we just matched a node that has no children,
1341 // we can stop searching. there should be no case in
1342 // which the node was not also marked as end of phrase.
1343 else
1344 break;
1345 }
1346
1347 if ((--sl) > 0) {
1348 // if we get here, we have discounted one child, but
1349 // we still have more children to examine from the last matched node.
1350 // we don't keep more than one current interesting node - no backtracking
1351 // is necessary, as there is only ever one occurrence of a given character as
1352 // a branch of a given node. backtracking would therefore never
1353 // trigger a match down a different route than has been taken thus far, so
1354 // don't bother.
1355 pos = graphdata[currnode + 4 + (graphdata[currnode + 2] - sl)];
1356 continue;
1357 }
1358 // if we get here, we've discounted all branches at this depth, and the search is over.
1359 break;
1360 }
1361 }
1362 }
1363 #ifdef DEBUG_LOW
1364 DEBUG_debug("Map start");
1365 for (std::map<std::string, std::pair<unsigned int, int> >::iterator i = result.begin(); i != result.end(); i++) {
1366 DEBUG_debug("Map: ", i->first, " ", i->second.second);
1367 }
1368 DEBUG_debug("Map end");
1369 #endif
1370 }
1371
1372 void ListContainer::graphAdd(String s, const int inx, int item) {
1373 unsigned char p = s.charAt(0);
1374 unsigned char c;
1375 bool found = false;
1376 String t;
1377 int i, px;
1378 int numlinks;
1379 int *graphdata;
1380 int *graphdata2 = realgraphdata + ROOTOFFSET;
1381 if (inx == 0)
1382 graphdata = realgraphdata;
1383 else
1384 graphdata = realgraphdata + ROOTOFFSET;
1385 //iterate over the input node's immediate children
1386 for (i = 0; i < graphdata[inx * GRAPHENTRYSIZE + 2]; i++) {
1387 //grab the character from this child
1388 c = (unsigned char) graphdata2[(graphdata[inx * GRAPHENTRYSIZE + 4 + i]) * GRAPHENTRYSIZE];
1389 if (p == c) {
1390 //it matches the first char of our string!
1391 //keep searching, starting from here, to see if the entire phrase is already in the graph
1392 t = s;
1393 t.lop();
1394 if (t.length() > 0) {
1395 graphAdd(t, graphdata[inx * GRAPHENTRYSIZE + 4 + i], item);
1396 return;
1397 }
1398 found = true;
1399
1400 // this means the phrase is already there
1401 // as part of an existing phrase
1402
1403 //check the end of word flag on the child
1404 px = graphdata2[(graphdata[inx * GRAPHENTRYSIZE + 4 + i]) * GRAPHENTRYSIZE + 1];
1405 if (px == 1) {
1406 // the exact phrase is already there
1407 px = graphdata2[(graphdata[inx * GRAPHENTRYSIZE + 4 + i]) * GRAPHENTRYSIZE + 3];
1408
1409 // -1=exception
1410 // 0=banned
1411 // 1=weighted
1412 // 10 = combination exception
1413 // 11 = combination banned
1414 // 12 = combination weighted
1415 // 20,21,22 = end of combi marker
1416 if ((itemtype[px] > 9 && itemtype[item] < 10) ||
1417 (itemtype[px] == 1 && itemtype[item] == 1 && (weight[item] > weight[px])) ||
1418 (itemtype[px] == 1 && itemtype[item] == 0) || itemtype[item] == -1) {
1419 // exists as a combi entry already
1420 // if got here existing entry must be a combi AND
1421 // new entry is not a combi so we overwrite the
1422 // existing values as combi values and types are
1423 // stored in the combilist
1424 // OR
1425 // both are weighted phrases and the new phrase is higher weighted
1426 // OR
1427 // the existing phrase is weighted and the new phrase is banned
1428 // OR
1429 // new phrase is an exception; exception phrases take precedence
1430 itemtype[px] = itemtype[item];
1431 weight[px] = weight[item];
1432 categoryindex[px] = categoryindex[item];
1433 timelimitindex[px] = timelimitindex[item];
1434 }
1435 }
1436 }
1437 }
1438 // the phrase wasn't already in the list, so add it
1439 if (!found) {
1440 i = graphitems;
1441 graphitems++;
1442 // Reallocate memory if we're running out
1443 if (current_graphdata_size < ((GRAPHENTRYSIZE * graphitems) + ROOTOFFSET)) {
1444 int new_current_graphdata_size = (GRAPHENTRYSIZE * (graphitems + 256)) + ROOTOFFSET;
1445 realgraphdata = (int *) realloc(realgraphdata, sizeof(int) * new_current_graphdata_size);
1446 if (realgraphdata == NULL) {
1447 E2LOGGER_error("Cannot reallocate memory for phrase tree: ", strerror(errno));
1448 exit(1);
1449 }
1450 memset(realgraphdata + current_graphdata_size, 0,
1451 sizeof(int) * (new_current_graphdata_size - current_graphdata_size));
1452 current_graphdata_size = new_current_graphdata_size;
1453 graphdata2 = realgraphdata + ROOTOFFSET;
1454 if (inx == 0)
1455 graphdata = realgraphdata;
1456 else
1457 graphdata = realgraphdata + ROOTOFFSET;
1458 }
1459 numlinks = graphdata[inx * GRAPHENTRYSIZE + 2];
1460 if ((inx == 0) ? ((numlinks + 1) > MAXROOTLINKS) : ((numlinks + 1) > MAXLINKS)) {
1461 E2LOGGER_error("Cannot load phraselists from this many languages/encodings simultaneously. (more than ", ((inx == 0) ? MAXROOTLINKS : MAXLINKS), " links from this node! [1])");
1462 exit(1);
1463 }
1464 if ((numlinks + 1) > maxchildnodes) {
1465 maxchildnodes = numlinks + 1;
1466 #ifdef DEBUG_LOW
1467 prolificroot = (inx == 0);
1468 #endif
1469 }
1470 #ifdef DEBUG_LOW
1471 else if ((numlinks + 1) > secondmaxchildnodes)
1472 secondmaxchildnodes = numlinks + 1;
1473 #endif
1474 graphdata[inx * GRAPHENTRYSIZE + 2]++;
1475 graphdata[inx * GRAPHENTRYSIZE + 4 + numlinks] = i;
1476 graphdata2[i * GRAPHENTRYSIZE] = p;
1477 graphdata2[i * GRAPHENTRYSIZE + 3] = item;
1478 s.lop();
1479 // iterate over remaining characters and add child nodes
1480 while (s.length() > 0) {
1481 numlinks = graphdata2[i * GRAPHENTRYSIZE + 2];
1482 if ((inx == 0) ? ((numlinks + 1) > MAXROOTLINKS) : ((numlinks + 1) > MAXLINKS)) {
1483 E2LOGGER_error("Cannot load phraselists from this many languages/encodings simultaneously. (more than ", ((inx == 0) ? MAXROOTLINKS : MAXLINKS), " links from this node! [2])" );
1484 exit(1);
1485 }
1486 if ((numlinks + 1) > maxchildnodes) {
1487 maxchildnodes = numlinks + 1;
1488 #ifdef DEBUG_LOW
1489 prolificroot = (inx == 0);
1490 #endif
1491 }
1492 #ifdef DEBUG_LOW
1493 else if ((numlinks + 1) > secondmaxchildnodes)
1494 secondmaxchildnodes = numlinks + 1;
1495 #endif
1496 graphdata2[i * GRAPHENTRYSIZE + 2]++;
1497 graphdata2[i * GRAPHENTRYSIZE + 4 + numlinks] = i + 1;
1498 i++;
1499 graphitems++;
1500 // Reallocate memory if we're running out
1501 if (current_graphdata_size < ((GRAPHENTRYSIZE * graphitems) + ROOTOFFSET)) {
1502 int new_current_graphdata_size = (GRAPHENTRYSIZE * (graphitems + 256)) + ROOTOFFSET;
1503 realgraphdata = (int *) realloc(realgraphdata, sizeof(int) * new_current_graphdata_size);
1504 if (realgraphdata == NULL) {
1505 E2LOGGER_error("Cannot reallocate memory for phrase tree: ", strerror(errno));
1506 exit(1);
1507 }
1508 memset(realgraphdata + current_graphdata_size, 0,
1509 sizeof(int) * (new_current_graphdata_size - current_graphdata_size));
1510 current_graphdata_size = new_current_graphdata_size;
1511 graphdata2 = realgraphdata + ROOTOFFSET;
1512 if (inx == 0)
1513 graphdata = realgraphdata;
1514 else
1515 graphdata = realgraphdata + ROOTOFFSET;
1516 }
1517 p = s.charAt(0);
1518 graphdata2[i * GRAPHENTRYSIZE] = p;
1519 graphdata2[i * GRAPHENTRYSIZE + 3] = item;
1520 s.lop();
1521 }
1522 graphdata2[i * GRAPHENTRYSIZE + 1] = 1;
1523 }
1524 }
1525
1526
1527 void ListContainer::addToItemList(const char *s, size_t len) {
1528 list.push_back(data_length);
1529 lengthlist.push_back(len);
1530 for (size_t i = 0; i < len; i++) {
1531 data[data_length + i] = s[i];
1532 }
1533 data[data_length + len] = 0;
1534 data_length += len + 1;
1535 items++;
1536 }
1537
1538 bool ListContainer::addToTimeList(String &line) {
1539 TimeLimit tl;
1540 if (readTimeBand(line, tl)) {
1541 timelist.push_back(tl);
1542 return true;
1543 }
1544 return false;
1545 }
1546
1547 void ListContainer::addToIPList(String &line) {
1548 if (is_map) return addToIPMap(line);
1549
1550 RegResult Rre;
1551
1552 // store the IP address (numerically, not as a string) and filter group in either the IP list or range list
1553 if (matchIP.match(line.toCharArray(), Rre)) {
1554 struct in_addr address;
1555 if (inet_aton(line.toCharArray(), &address)) {
1556 uint32_t addr = ntohl(address.s_addr);
1557 iplist.push_back(addr);
1558 }
1559 } else if (matchSubnet.match(line.toCharArray(), Rre)) {
1560 struct in_addr address;
1561 struct in_addr addressmask;
1562 String subnet(line.before("/"));
1563 String mask(line.after("/"));
1564 if (inet_aton(subnet.toCharArray(), &address) && inet_aton(mask.toCharArray(), &addressmask)) {
1565 ipl_rangestruct s;
1566 s.startaddr = ntohl(address.s_addr);
1567 uint32_t imask = ntohl(addressmask.s_addr);
1568 s.endaddr = s.startaddr | ~imask;
1569 iprangelist.push_back(s);
1570 }
1571 } else if (matchCIDR.match(line.toCharArray(), Rre)) {
1572 struct in_addr address;
1573 struct in_addr addressmask;
1574 String subnet(line.before("/"));
1575 String cidr(line.after("/"));
1576 int m = cidr.toInteger();
1577 int host_part = 32 - m;
1578 if (host_part > -1) {
1579 String mask = (0xFFFFFFFF << host_part);
1580 if (inet_aton(subnet.toCharArray(), &address) && inet_aton(mask.toCharArray(), &addressmask)) {
1581 ipl_rangestruct s;
1582 s.startaddr = ntohl(address.s_addr);
1583 uint32_t imask = ntohl(addressmask.s_addr);
1584 s.endaddr = s.startaddr | ~imask;
1585 iprangelist.push_back(s);
1586 }
1587 }
1588 } else if (matchRange.match(line.toCharArray(), Rre)) {
1589 struct in_addr addressstart;
1590 struct in_addr addressend;
1591 String start(line.before("-"));
1592 String end(line.after("-"));
1593 if (inet_aton(start.toCharArray(), &addressstart) && inet_aton(end.toCharArray(), &addressend)) {
1594 ipl_rangestruct r;
1595 r.startaddr = ntohl(addressstart.s_addr);
1596 r.endaddr = ntohl(addressend.s_addr);
1597 iprangelist.push_back(r);
1598 }
1599 else
1600 {
1601 DEBUG_config("Not adding to any IP list:", line);
1602 }
1603 }
1604 }
1605
1606 void ListContainer::addToDataMap(String &line) {
1607 String key, value;
1608
1609 // split into key & value
1610 if (line.contains("=")) {
1611 key = line.before("=");
1612 key.removeWhiteSpace();
1613 value = line.after("=");
1614 value.removeWhiteSpace();
1615 if (value.startsWith("filter"))
1616 value = value.after("filter");
1617 } else {
1618 E2LOGGER_error("No filter group given; entry %s in %s", line.toCharArray(), sourcefile.c_str());
1619 //warn = true;
1620 return;
1621 }
1622
1623 DEBUG_debug("key: ", key, " value: ", value);
1624
1625 datamap d(key,value);
1626 datamaplist.push_back(d);
1627 }
1628
1629 void ListContainer::addToIPMap(String &line) {
1630 RegResult Rre;
1631 String key, value;
1632
1633 // split into key & value
1634 if (line.contains("=")) {
1635 key = line.before("=");
1636 key.removeWhiteSpace();
1637 value = line.after("=");
1638 value.removeWhiteSpace();
1639 if (value.startsWith("filter"))
1640 value = value.after("filter");
1641 } else {
1642 E2LOGGER_error("No filter group given; entry %s in %s", line, sourcefile );
1643 //warn = true;
1644 return;
1645 }
1646 DEBUG_debug("key: ", key, " value: ", value);
1647 if ((value.toInteger() < 1) || (value.toInteger() > o.filter.filter_groups)) {
1648 E2LOGGER_error("Filter group out of range; entry ", line, " in ", sourcefile);
1649 //warn = true;
1650 return;
1651 }
1652
1653 // store the IP address (numerically, not as a string) and filter group in either the IP list, subnet list or range list
1654 if (matchIP.match(key.toCharArray(), Rre)) {
1655 // std::cerr << "Is straigth IP " << key << std::endl;
1656 struct in_addr address;
1657 auto aton_res = inet_aton(key.toCharArray(), &address);
1658 // std::cerr << "aton returned " << aton_res << std::endl;
1659 if (aton_res != 0) {
1660 ipmap tmap(ntohl(address.s_addr), value);
1661 ipmaplist.push_back(tmap);
1662 }
1663 } else if (matchSubnet.match(key.toCharArray(), Rre)) {
1664 // std::cerr << "Is subnet IP " << key << std::endl;
1665 struct in_addr address;
1666 struct in_addr addressmask;
1667 String subnet(key.before("/"));
1668 String mask(key.after("/"));
1669 if (inet_aton(subnet.toCharArray(), &address) && inet_aton(mask.toCharArray(), &addressmask)) {
1670 rangestruct s;
1671 s.startaddr = ntohl(address.s_addr);
1672 uint32_t imask = ntohl(addressmask.s_addr);
1673 s.endaddr = s.startaddr | ~imask;
1674 s.group = value;
1675 ipmaprangelist.push_back(s);
1676 }
1677 } else if (matchCIDR.match(key.toCharArray(), Rre)) {
1678 // std::cerr << "Is CIDR " << key << std::endl;
1679 struct in_addr address;
1680 struct in_addr addressmask;
1681 String subnet(key.before("/"));
1682 String cidr(key.after("/"));
1683 int m = cidr.toInteger();
1684 int host_part = 32 - m;
1685 if (host_part > -1) {
1686 String mask = (0xFFFFFFFF << host_part);
1687 if (inet_aton(subnet.toCharArray(), &address) && inet_aton(mask.toCharArray(), &addressmask)) {
1688 rangestruct s;
1689 s.startaddr = ntohl(address.s_addr);
1690 uint32_t imask = ntohl(addressmask.s_addr);
1691 s.endaddr = s.startaddr | ~imask;
1692 s.group = value;
1693 ipmaprangelist.push_back(s);
1694 }
1695 }
1696 } else if (matchRange.match(key.toCharArray(), Rre)) {
1697 // std::cerr << "Is IP range " << key << std::endl;
1698 struct in_addr addressstart;
1699 struct in_addr addressend;
1700 String start(key.before("-"));
1701 String end(key.after("-"));
1702 if (inet_aton(start.toCharArray(), &addressstart) && inet_aton(end.toCharArray(), &addressend)) {
1703 rangestruct r;
1704 r.startaddr = ntohl(addressstart.s_addr);
1705 r.endaddr = ntohl(addressend.s_addr);
1706 r.group = value;
1707 ipmaprangelist.push_back(r);
1708 }
1709 }
1710 // hmmm. the key didn't match any of our regular expressions. output message & return a warning value.
1711 else {
1712 E2LOGGER_error( "Entry ", line, " in ", sourcefile, " was not recognised as an IP address, subnet or range");
1713 //warn = true;
1714 }
1715 }
1716
1717 // binary search list for given IP & return filter group, or -1 on failure
1718 String ListContainer::searchIPMap(int a, int s, const uint32_t &ip) {
1719 // change to serial search for testing
1720 if (false) {
1721 for (auto item : ipmaplist) {
1722 if (item.addr == ip) {
1723 return item.group;
1724 }
1725 }
1726 return "";
1727 }
1728
1729 if (true) {
1730 if (a > s)
1731 return "";
1732 int m = (a + s) / 2;
1733 if (ipmaplist[m].addr == ip)
1734 return ipmaplist[m].group;
1735 if (ipmaplist[m].addr < ip)
1736 return searchIPMap(m + 1, s, ip);
1737 if (a == s)
1738 return "";
1739 return searchIPMap(a, m - 1, ip);
1740 }
1741 }
1742
1743 // search subnet list for given IP & return filter group or -1
1744 //String ListContainer::inSubnetMap(const uint32_t &ip) {
1745 //for (std::list<subnetstruct>::const_iterator i = ipmapsubnetlist.begin(); i != ipmapsubnetlist.end(); ++i) {
1746 //if (i->maskedaddr == (ip & i->mask)) {
1747 //return i->group;
1748 //}
1749 //}
1750 //return "";
1751 //}
1752
1753 // search range list for a range containing given IP & return filter group or -1
1754 String ListContainer::inIPRangeMap(const uint32_t &ip) {
1755 if (!ipmaprangelist.empty())
1756 {
1757 rangestruct t;
1758 t.startaddr = ip;
1759 auto one_above = std::upper_bound(ipmaprangelist.begin(), ipmaprangelist.end(),t);
1760 if (one_above != ipmaprangelist.begin()) {
1761 auto i = one_above;
1762 i--; // move pointer to record which is the highest value that is less or equal to ip.
1763 if ((ip >= i->startaddr) && (ip <= i->endaddr)) {
1764 return i->group;
1765 }
1766 }
1767 }
1768 return "";
1769 }
1770
1771 String ListContainer::inIPMap(const uint32_t &ip) {
1772 if (ipmaplist.size() > 0) {
1773 return searchIPMap(0, ipmaplist.size() - 1, ip);
1774 }
1775 return "";
1776 }
1777
1778 // binary search list for given key & return filter group, or -1 on failure
1779 String ListContainer::searchDataMap(int a, int s, const String &key) {
1780 // change to serial search for testing
1781 if (false) {
1782 for (auto item : datamaplist) {
1783 if (item.key== key) {
1784 return item.group;
1785 }
1786 }
1787 return "";
1788 }
1789
1790 if (true) {
1791 if (a > s)
1792 return "";
1793 int m = (a + s) / 2;
1794 if (datamaplist[m].key == key)
1795 return datamaplist[m].group;
1796 if (datamaplist[m].key < key)
1797 return searchDataMap(m + 1, s, key);
1798 if (a == s)
1799 return "";
1800 return searchDataMap(a, m - 1, key);
1801 }
1802 }
1803
1804 bool ListContainer::getMapData(String &key, String &match, String &result) {
1805 if(datamaplist.empty())
1806 return false;
1807 result = searchDataMap(0,datamaplist.size() - 1,key);
1808 return true;
1809 }
1810
1811 String ListContainer::getIPMapData(std::string &ip) {
1812 struct in_addr sin;
1813 inet_aton(ip.c_str(), &sin);
1814 uint32_t addr = ntohl(sin.s_addr);
1815 String fgs;
1816 String rfg;
1817 // check straight IPs, subnets, and ranges
1818 fgs = inIPMap(addr);
1819 if (fgs != "") {
1820 rfg = fgs;
1821 DEBUG_debug("Matched IP ", ip, " to straight IP list");
1822 return rfg;
1823 }
1824 //fgs = inSubnetMap(addr);
1825 //if (fgs != "") {
1826 //rfg = fgs;
1827 //DEBUG_debug("Matched IP ", ip, " to subnet");
1828 //return rfg;
1829 //}
1830 fgs = inIPRangeMap(addr);
1831 if (fgs != "") {
1832 rfg = fgs;
1833 DEBUG_debug("Matched IP ", ip, " to range");
1834 return rfg;
1835 }
1836 DEBUG_debug("Matched IP ", ip, " to nothing");
1837 return "";
1838 }
1839
1840 // Note: searches a reverse ordered (descending) list
1841 int ListContainer::search(int (ListContainer::*comparitor)(const char *a, const char *b), int a, int s, const char *p) {
1842 if (a > s)
1843 return (-1 - a);
1844 int m = (a + s) / 2;
1845 int r = (this->*comparitor)(p, data + list[m]);
1846 if (r == 0)
1847 return m;
1848 if (r == -1)
1849 return search(comparitor, m + 1, s, p);
1850 if (a == s)
1851 return (-1 - a);
1852 return search(comparitor, a, m - 1, p);
1853 }
1854
1855 // The following comparator function will result in a search within a list sorted in descending order - which is counter-intuitive!!!
1856 int ListContainer::greaterThanEWF(const char *a, const char *b) {
1857 int alen = strlen(a);
1858 int blen = strlen(b);
1859 int apos = alen - 1;
1860 int bpos = blen - 1;
1861 for (int maxlen = (alen < blen ? alen : blen); maxlen > 0; apos--, bpos--, maxlen--)
1862 if (a[apos] > b[bpos])
1863 return 1;
1864 else if (a[apos] < b[bpos])
1865 return -1;
1866 if (alen > blen)
1867 return 1;
1868 else if (alen < blen)
1869 return -1;
1870 return 0; // both equal
1871 }
1872
1873 // The following comparator function will result in a search within a list sorted in descending order - which is counter-intuitive!!!
1874 int ListContainer::greaterThanSWF(const char *a, const char *b) {
1875 int alen = strlen(a);
1876 int blen = strlen(b);
1877 int maxlen = alen < blen ? alen : blen;
1878 for (int i = 0; i < maxlen; i++)
1879 if (a[i] > b[i])
1880 return 1;
1881 else if (a[i] < b[i])
1882 return -1;
1883 if (alen > blen)
1884 return 1;
1885 else if (alen < blen)
1886 return -1;
1887 return 0; // both equal
1888 }
1889
1890 int ListContainer::greaterThanSW(const char *a, const char *b) {
1891 int alen = strlen(a);
1892 int blen = strlen(b);
1893 int maxlen = alen < blen ? alen : blen;
1894 for (int i = 0; i < maxlen; i++)
1895 if (a[i] > b[i])
1896 return 1;
1897 else if (a[i] < b[i])
1898 return -1;
1899
1900 // if the URLs didn't match and the one the user is browsing to is longer
1901 // than what we just compared against, we need to compare against longer URLs,
1902 // but only if the next character is actually part of a folder name rather than a separator.
1903 // ('cos if it *is* a separator, it doesn't matter that the overall URL is longer, the
1904 // beginning of it matches a banned URL.)
1905 if ((alen > blen) && !(a[blen] == '/' || a[blen] == '?' || a[blen] == '&' || a[blen] == '='))
1906 return 1;
1907
1908 // if the banned URL is longer than the URL we're checking, the two
1909 // can't possibly match.
1910 else if (blen > alen)
1911 return -1;
1912
1913 return 0; // both equal
1914 }
1915
1916 int ListContainer::greaterThanEW(const char *a, const char *b) {
1917 int alen = strlen(a);
1918 int blen = strlen(b);
1919 int apos = alen - 1;
1920 int bpos = blen - 1;
1921 for (int maxlen = (alen < blen ? alen : blen); maxlen > 0; apos--, bpos--, maxlen--)
1922 if (a[apos] > b[bpos])
1923 return 1;
1924 else if (a[apos] < b[bpos])
1925 return -1;
1926 if (blen > alen)
1927 return -1;
1928 return 0; // both equal
1929 }
1930
1931 bool ListContainer::increaseMemoryBy(size_t bytes) {
1932 if (data_memory > 0) {
1933 data = (char *) realloc(data, (data_memory + bytes) * sizeof(char));
1934 if (data == nullptr)
1935 return false;
1936 memset(data + data_memory, 0, bytes * sizeof(char));
1937 data_memory += bytes;
1938 } else {
1939 free(data);
1940 data = (char *) calloc(bytes, sizeof(char));
1941 if (data == nullptr)
1942 return false;
1943 memset(data + data_memory, 0, bytes * sizeof(char));
1944 data_memory = bytes;
1945 }
1946 return true;
1947 }
1948
1949 size_t getFileLength(const char *filename) {
1950 struct stat status;
1951 int rc = stat(filename, &status);
1952 if (rc < 0)
1953 throw std::runtime_error(strerror(errno));
1954 return status.st_size;
1955 }
1956
1957 time_t getFileDate(const char *filename) {
1958 struct stat status;
1959 int rc = stat(filename, &status);
1960 if (rc != 0) {
1961 if (errno == ENOENT) {
1962 E2LOGGER_error("*List Error* opening ",filename, ":", strerror(errno));
1963 return 0;
1964 }
1965 // If there are permission problems, just reload the file (CN)
1966 if (errno == EACCES) {
1967 E2LOGGER_error("*List Error* opening ", filename, ". Check directory and file permissions and ownership. They should be 750 and 640 and readable by the e2guardian user: ", strerror(errno));
1968 return 0;
1969 }
1970 else {
1971 E2LOGGER_error("*List Error* opening ", filename, ". Check directory and file permissions and ownership. They should be 750 and 640 and readable by the e2guardian user: ", strerror(errno));
1972 return 0;
1973 //return sysv_kill(o.pid_filename);
1974 }
1975 }
1976 return status.st_mtime;
1977 }
1978
1979 #ifdef NODEF
1980 time_t getFileDate(const char *filename)
1981 {
1982 struct stat status;
1983 int rc = stat(filename, &status);
1984 if (rc != 0) {
1985 if (errno == ENOENT)
1986 return 0;
1987 else
1988 throw std::runtime_error(strerror(errno));
1989 }
1990 return status.st_mtime;
1991 }
1992 #endif
1993
1994 bool ListContainer::upToDate() {
1995 if (sourcefile.startsWith("memory:"))
1996 return true;
1997
1998 if (getFileDate(sourcefile.toCharArray()) > filedate) {
1999 return false;
2000 }
2001
2002 for (unsigned int i = 0; i < morelists.size(); i++) {
2003 if (!(*o.lm.l[morelists[i]]).upToDate()) {
2004 return false;
2005 }
2006 }
2007 return true;
2008 }
2009
2010 bool ListContainer::readTimeTag(String *tag, TimeLimit &tl) {
2011 DEBUG_config("Found a time tag");
2012 String temp((*tag).after("#time: "));
2013 return readTimeBand(temp, tl);
2014 }
2015
2016 bool ListContainer::readTimeBand(String &tag, TimeLimit &tl) {
2017 String temp(tag);
2018 unsigned int tsthour, tstmin, tendhour, tendmin;
2019 temp.removeWhiteSpace();
2020 tsthour = temp.before(" ").toInteger();
2021 temp = temp.after(" ");
2022 temp.removeWhiteSpace();
2023 tstmin = temp.before(" ").toInteger();
2024 temp = temp.after(" ");
2025 temp.removeWhiteSpace();
2026 tendhour = temp.before(" ").toInteger();
2027 temp = temp.after(" ");
2028 temp.removeWhiteSpace();
2029 tendmin = temp.before(" ").toInteger();
2030 String tdays(temp.after(" "));
2031 tdays.removeWhiteSpace();
2032 if (tsthour > 23) {
2033 E2LOGGER_error("Time Tag Start Hour over bounds.");
2034 return false;
2035 }
2036 if (tendhour > 23) {
2037 E2LOGGER_error("Time Tag End Hour over bounds.");
2038 return false;
2039 }
2040 if (tstmin > 59) {
2041 E2LOGGER_error("Time Tag Start Min over bounds.");
2042 return false;
2043 }
2044 if (tendmin > 59) {
2045 E2LOGGER_error("Time Tag End Min over bounds.");
2046 return false;
2047 }
2048 if (tdays.length() > 7) {
2049 E2LOGGER_error("Time Tag Days over bounds.");
2050 return false;
2051 }
2052 istimelimited = true;
2053 tl.sthour = tsthour;
2054 tl.stmin = tstmin;
2055 tl.endhour = tendhour;
2056 tl.endmin = tendmin;
2057 tl.days = tdays;
2058 tl.timetag = tag;
2059 return true;
2060 }
2061
2062 // Returns true if the current time is within the limits specified on this list.
2063 // For phrases, the time limit list index must be passed in -
2064 // included lists don't have their own ListContainer, so time limits are stored differently.
2065 bool ListContainer::isNow(int index) {
2066 if (!istimelimited) {
2067 return true;
2068 }
2069 TimeLimit tl = listtimelimit;
2070 if (index > -1) {
2071 tl = timelimits[index];
2072 }
2073 return isNow(tl);
2074 }
2075
2076 // Used for timelists only - returns true if current time is in list
2077 bool ListContainer::isNowInTimelist() {
2078 if (timelist.size() > 0) {
2079 for (unsigned int i = 0; i < timelist.size(); i++) {
2080 if (isNow(timelist[i]))
2081 return true;
2082 }
2083 }
2084 return false;
2085 }
2086
2087 bool ListContainer::isNow(TimeLimit &tl) {
2088 time_t tnow; // to hold the result from time()
2089 struct tm *tmnow; // to hold the result from localtime()
2090 unsigned int hour, min, wday;
2091 time(&tnow); // get the time after the lock so all entries in order
2092 tmnow = localtime(&tnow); // convert to local time (BST, etc)
2093 hour = tmnow->tm_hour;
2094 min = tmnow->tm_min;
2095 wday = tmnow->tm_wday;
2096 // wrap week to start on Monday
2097 if (wday == 0) {
2098 wday = 7;
2099 }
2100 wday--;
2101 unsigned char cday = '0' + wday;
2102 bool matchday = false;
2103 for (unsigned int i = 0; i < tl.days.length(); i++) {
2104 if (tl.days[i] == cday) {
2105 matchday = true;
2106 break;
2107 }
2108 }
2109 if (!matchday) {
2110 return false;
2111 }
2112 if (hour < tl.sthour) {
2113 return false;
2114 }
2115 if (hour > tl.endhour) {
2116 return false;
2117 }
2118 if (hour == tl.sthour) {
2119 if (min < tl.stmin) {
2120 return false;
2121 }
2122 }
2123 if (hour == tl.endhour) {
2124 if (min > tl.endmin) {
2125 return false;
2126 }
2127 }
2128 DEBUG_debug("time match ", tl.sthour, ":", tl.stmin, "-", tl.endhour, ":", tl.endmin, " ", hour, ":", min, " ", sourcefile);
2129 return true;
2130 }
2131
2132 int ListContainer::getCategoryIndex(String *lcat) {
2133 // where in the category list is our category? if nowhere, add it.
2134 if ((*lcat).length() < 2) {
2135 DEBUG_debug("blank entry index");
2136 return 0; // blank entry index
2137 }
2138 int l = (signed) listcategory.size();
2139 int i;
2140 for (i = 0; i < l; i++) {
2141 if ((*lcat) == listcategory[i]) {
2142 return i;
2143 }
2144 }
2145 listcategory.push_back((*lcat));
2146 return l;
2147 }
2148
2149 String ListContainer::getListCategoryAt(unsigned int index, unsigned int *catindex) {
2150 //category index of -1 indicates uncategorised list
2151 if ((index >= categoryindex.size()) || (categoryindex[index] < 0)) {
2152 return "";
2153 }
2154 //return category index (allows naughtyfilter to do numeric comparison between categories
2155 //when doing duplicate search; much faster than string comparison)
2156 if (catindex != NULL) {
2157 (*catindex) = categoryindex[index];
2158 }
2159 return listcategory[categoryindex[index]];
2160 }
2161
2162 String ListContainer::getListCategoryAtD(unsigned int index) {
2163 //category index of -1 indicates uncategorised list
2164 if ((index < 0) || (index >= listcategory.size())) {
2165 return "";
2166 }
2167 return listcategory[index];
2168 }
2169
2170
2171 // search for IP in list of individual IPs, ranges, subnets
2172 bool ListContainer::inIPList(const std::string &ipstr, String &match) {
2173 // const char *ListContainer::inIPList(const std::string &ipstr) {
2174 struct in_addr addr;
2175 inet_aton(ipstr.c_str(), &addr);
2176 uint32_t ip = ntohl(addr.s_addr);
2177
2178 // start with individual IPs
2179 if ((iplist.size() > 0) && std::binary_search(iplist.begin(), iplist.end(), ip)) {
2180 // only return a hostname if that's what we matched against
2181 match = hIPtoChar(ip);
2182 return true;
2183 }
2184
2185 // ranges
2186 if (!iprangelist.empty())
2187 {
2188 ipl_rangestruct t;
2189 t.startaddr = ip;
2190 auto one_above = std::upper_bound(iprangelist.begin(), iprangelist.end(),t);
2191 if (one_above != iprangelist.begin()) {
2192 auto i = one_above;
2193 i--; // move pointer to record which is the highest value that is less or equal to ip.
2194 if ((ip >= i->startaddr) && (ip <= i->endaddr)) {
2195 match = hIPtoChar(i->startaddr);
2196 match += "-";
2197 match += hIPtoChar(i->endaddr);
2198 return true;
2199 }
2200 }
2201 }
2202 //if (iprangelist.size() > 0) {
2203 //for (std::vector<ipl_rangestruct>::const_iterator i = iprangelist.begin(); i != iprangelist.end(); ++i) {
2204 //if ((ip >= i->startaddr) && (ip <= i->endaddr)) {
2205 //String ret = hIPtoChar(i->startaddr);
2206 //ret += "-";
2207 //ret += hIPtoChar(i->endaddr);
2208 //match = ret;
2209 //return true;
2210 //}
2211 //}
2212 //}
2213
2214 // subnets
2215 //if (ipsubnetlist.size() > 0) {
2216 //for (std::list<ipl_subnetstruct>::const_iterator i = ipsubnetlist.begin(); i != ipsubnetlist.end(); ++i) {
2217 //if (i->maskedaddr == (ip & i->mask)) {
2218 //String ret = hIPtoChar(i->maskedaddr);
2219 //ret += "/";
2220 //ret += hIPtoChar(i->mask);
2221 //match = ret;
2222 //return true;
2223 //}
2224 //}
2225 //}
2226
2227 DEBUG_debug("inIPList ", category, " no match for ", ipstr);
2228 return false;
2229 }
2230
2231 const char *ListContainer::hIPtoChar(uint32_t ip) {
2232 struct in_addr addr;
2233 addr.s_addr = htonl(ip);
2234 return inet_ntoa(addr);
2235 }
2236
2237 void ListContainer::dump_data() {
2238 DEBUG_debug("List size is ",list.size());
2239
2240 if(list.size() > 0) {
2241 for (std::vector<size_t>::const_iterator i = list.begin(); i != list.end(); ++i) {
2242 DEBUG_debug("LD:",data + *i);
2243 }
2244 }
2245 }
2246
2247 bool ListContainer::self_check() {
2248 DEBUG_debug("List size is ", list.size());
2249 int ok_cnt = 0, cnt = 0;
2250
2251 if (list.size() > 0) {
2252 for (std::vector<size_t>::const_iterator i = list.begin(); i != list.end(); ++i) {
2253 // DEBUG_debug("LD:", data + *i);
2254 int r;
2255 if (isSW) {
2256 r = search(&ListContainer::greaterThanSWF, 0, items - 1, data + *i);
2257 } else {
2258 r = search(&ListContainer::greaterThanEWF, 0, items - 1, data + *i);
2259 }
2260 if (r >= 0) {
2261 ok_cnt++;
2262 }
2263 cnt++;
2264 }
2265 if (ok_cnt < cnt) {
2266 DEBUG_config("LC: NOTOK ", ok_cnt, "/", cnt);
2267 return true;
2268 } else {
2269 DEBUG_config("LC: OK ", ok_cnt, "/", cnt);
2270 return false;
2271 }
2272 }
2273 return true;
2274 }