Debian Codegraph @

addon/doxysearch/doxysearch.cpp

raw
1 /******************************************************************************
2 *
3 * Copyright (C) 1997-2022 by Dimitri van Heesch.
4 *
5 * Permission to use, copy, modify, and distribute this software and its
6 * documentation under the terms of the GNU General Public License is hereby
7 * granted. No representations are made about the suitability of this software
8 * for any purpose. It is provided "as is" without express or implied warranty.
9 * See the GNU General Public License for more details.
10 *
11 * Documents produced by Doxygen are derivative works derived from the
12 * input used in their production; they are not affected by this license.
13 *
14 */
15
16 // STL includes
17 #include <cstdio>
18 #include <cstdlib>
19 #include <string>
20 #include <vector>
21 #include <sstream>
22 #include <iostream>
23 #include <string>
24 #include <algorithm>
25 #include <filesystem>
26
27 // Xapian includes
28 #include <xapian.h>
29
30 #include "version.h"
31
32
33 constexpr Xapian::valueno FIELD_TYPE{ 1 };
34 constexpr Xapian::valueno FIELD_NAME{ 2 };
35 constexpr Xapian::valueno FIELD_ARGS{ 3 };
36 constexpr Xapian::valueno FIELD_TAG{ 4 };
37 constexpr Xapian::valueno FIELD_URL{ 5 };
38 constexpr Xapian::valueno FIELD_KEYW{ 6 };
39 constexpr Xapian::valueno FIELD_DOC{ 7 };
40
41 constexpr int hex2dec(unsigned char c)
42 {
43 if (c >= '0' && c <= '9') return c - '0';
44 if (c >= 'a' && c <= 'f') return c - 'a' + 10;
45 if (c >= 'A' && c <= 'F') return c - 'A' + 10;
46 return -1;
47 }
48
49
50 bool dirExists(const std::string &dirname)
51 {
52 std::error_code ec;
53 return std::filesystem::is_directory(dirname, ec);
54 }
55
56
57 /** decodes a URI encoded string into a normal string. */
58 static std::string uriDecode(const std::string & sSrc)
59 {
60 // Note from RFC1630: "Sequences which start with a percent
61 // sign but are not followed by two hexadecimal characters
62 // (0-9, A-F) are reserved for future extension"
63
64 const unsigned char * pSrc = (const unsigned char *)sSrc.c_str();
65 const size_t SRC_LEN = sSrc.length();
66 const unsigned char * const SRC_END = pSrc + SRC_LEN;
67 // last decodable '%'
68 const unsigned char * const SRC_LAST_DEC = SRC_END - 2;
69
70 char * const pStart = new char[SRC_LEN];
71 char * pEnd = pStart;
72
73 while (pSrc < SRC_LAST_DEC)
74 {
75 if (*pSrc == '%') // replace %2A with corresponding ASCII character
76 {
77 int dec1, dec2;
78 unsigned char c1=*(pSrc+1);
79 unsigned char c2=*(pSrc+2);
80 if (-1 != (dec1 = hex2dec(c1)) && -1 != (dec2 = hex2dec(c2)))
81 {
82 *pEnd++ = (dec1 << 4) + dec2;
83 pSrc += 3;
84 continue;
85 }
86 }
87 else if (*pSrc == '+') // replace '+' with space
88 {
89 *pEnd++ = ' '; pSrc++;
90 continue;
91 }
92 *pEnd++ = *pSrc++;
93 }
94
95 // the last 2- chars
96 while (pSrc < SRC_END) *pEnd++ = *pSrc++;
97
98 std::string sResult(pStart, pEnd);
99 delete [] pStart;
100 return sResult;
101 }
102
103 /** return list of strings that result when splitting \a s using
104 * delimiter \a delim
105 */
106 static std::vector<std::string> split(const std::string &s, char delim)
107 {
108 std::vector<std::string> elems;
109 std::stringstream ss(s);
110 std::string item;
111 while (getline(ss, item, delim)) elems.push_back(item);
112 return elems;
113 }
114
115 /** Read type T from string \a s */
116 template<class T>
117 T fromString(const std::string& s)
118 {
119 std::istringstream stream (s);
120 T t;
121 stream >> t;
122 return t;
123 }
124
125 /** Class that holds the starting position of a word */
126 struct WordPosition
127 {
128 WordPosition(size_t s,size_t i) : start(s), index(i) {}
129 size_t start;
130 size_t index;
131 };
132
133 /** Class representing the '<' operator for WordPosition objects based on position. */
134 struct WordPosition_less
135 {
136 bool operator()(const WordPosition &p1,const WordPosition &p2) const
137 {
138 return p1.start<p2.start;
139 }
140 };
141
142 /** Class that holds a text fragment */
143 struct Fragment
144 {
145 Fragment(const std::string &t,int occ) : text(t), occurrences(occ) {}
146 std::string text;
147 int occurrences;
148 };
149
150 /** Class representing the '>' operator for Fragment objects based on occurrence. */
151 struct Fragment_greater
152 {
153 bool operator()(const Fragment &p1,const Fragment &p2) const
154 {
155 return p1.occurrences>p2.occurrences;
156 }
157 };
158
159 /** Class representing a range within a string */
160 struct Range
161 {
162 Range(size_t s,size_t e) : start(s), end(e) {}
163 size_t start;
164 size_t end;
165 };
166
167 /** Returns true if [start..start+len] is inside one of the \a ranges. */
168 static bool insideRange(const std::vector<Range> &ranges,size_t start,size_t len)
169 {
170 for (std::vector<Range>::const_iterator it = ranges.begin();
171 it!=ranges.end(); ++it
172 )
173 {
174 Range r = *it;
175 if (start>=r.start && start+len<r.end)
176 {
177 return true;
178 }
179 }
180 return false;
181 }
182
183 /** Returns a list of text \a fragments from \a s containing one or
184 * more \a words. The list is sorted according to the
185 * number of occurrences of words within the fragment.
186 */
187 static void highlighter(const std::string &s,
188 const std::vector<std::string> &words,
189 std::vector<Fragment> &fragments)
190 {
191 const std::string spanStart="<span class=\"hl\">";
192 const std::string spanEnd="</span>";
193 const std::string dots="...";
194 const size_t fragLen = 60;
195 size_t sl=s.length();
196
197 // find positions of words in s
198 int j=0;
199 std::vector<WordPosition> positions;
200 for (std::vector<std::string>::const_iterator it=words.begin();
201 it!=words.end();
202 ++it,++j
203 )
204 {
205 size_t pos=0;
206 size_t i;
207 std::string word = *it;
208 while ((i=s.find(word,pos))!=std::string::npos)
209 {
210 positions.push_back(WordPosition(i,j));
211 pos=i+word.length();
212 }
213 }
214 // sort on position
215 std::sort(positions.begin(),positions.end(),WordPosition_less());
216 // get fragments around words
217 std::vector<Range> ranges;
218 for (std::vector<WordPosition>::const_iterator it=positions.begin();
219 it!=positions.end();
220 ++it)
221 {
222 WordPosition wp = *it;
223 std::string w = words[wp.index];
224 size_t i=wp.start;
225 size_t wl=w.length();
226 if (!insideRange(ranges,i,wl))
227 {
228 if (wl>fragLen)
229 {
230 fragments.push_back(Fragment(spanStart+w+spanEnd,1));
231 ranges.push_back(Range(i,i+wl));
232 }
233 else
234 {
235 std::string startFragment,endFragment;
236 int bi=static_cast<int>(i)-static_cast<int>((fragLen-wl)/2);
237 size_t ei=i+wl+(fragLen-wl)/2;
238 int occ=0;
239 if (bi<0) { ei-=bi; bi=0; } else startFragment=dots;
240 if (ei>sl) { ei=sl; } else endFragment=dots;
241 while (bi>0 && !isspace(s[bi])) bi--; // round to start of the word
242 while (ei<sl && !isspace(s[ei])) ei++; // round to end of the word
243 // highlight any word in s between indexes bi and ei
244 std::string fragment=startFragment;
245 size_t pos=bi;
246 for (std::vector<WordPosition>::const_iterator it2=positions.begin();
247 it2!=positions.end();
248 ++it2)
249 {
250 WordPosition wp2 = *it2;
251 std::string w2 = words[wp2.index];
252 size_t wl2 = w2.length();
253 if (wp2.start>=bi && wp2.start+wl2<=ei) // word is inside the range!
254 {
255 fragment+=s.substr(pos,wp2.start-pos)+
256 spanStart+
257 s.substr(wp2.start,wl2)+
258 spanEnd;
259 pos=wp2.start+wl2;
260 occ++;
261 }
262 }
263 fragment+=s.substr(pos,ei-pos)+endFragment;
264 fragments.push_back(Fragment(fragment,occ));
265 ranges.push_back(Range(bi,ei));
266 }
267 }
268 }
269 std::sort(fragments.begin(),fragments.end(),Fragment_greater());
270 }
271
272 /** Escapes a string such that is can be included in a JSON structure */
273 static std::string escapeString(const std::string &s)
274 {
275 std::stringstream dst;
276 for (size_t i=0;i<s.length();i++)
277 {
278 char ch = s[i];
279 switch (ch)
280 {
281 case '\"': dst << "\\\""; break;
282 default: dst << ch; break;
283 }
284 }
285 return dst.str();
286 }
287
288 static void showError(const std::string &callback,const std::string &error)
289 {
290 std::cout << callback << "({\"error\":\"" << error << "\"})";
291 exit(0);
292 }
293
294 static void usage(const char *name, int exitVal = 1)
295 {
296 std::cerr << "Usage: " << name << "[query_string]" << std::endl;
297 std::cerr << " " << "alternatively the query string can be given by the environment variable QUERY_STRING" << std::endl;
298 exit(exitVal);
299 }
300
301 /** Main routine */
302 int main(int argc,char **argv)
303 {
304 // process inputs that were passed to us via QUERY_STRING
305 std::string callback;
306 try
307 {
308 std::string queryString;
309 if (argc == 1)
310 {
311 const char *queryEnv = getenv("QUERY_STRING");
312 if (queryEnv)
313 {
314 queryString = queryEnv;
315 }
316 else
317 {
318 usage(argv[0]);
319 }
320 }
321 else if (argc == 2)
322 {
323 if (std::string(argv[1])=="-h" || std::string(argv[1])=="--help")
324 {
325 usage(argv[0],0);
326 }
327 else if (std::string(argv[1])=="-v" || std::string(argv[1])=="--version")
328 {
329 std::cerr << argv[0] << " version: " << getFullVersion() << std::endl;
330 exit(0);
331 }
332 else
333 {
334 queryString = argv[1];
335 }
336 }
337 else
338 {
339 usage(argv[0]);
340 }
341
342 std::cout << "Content-Type:application/javascript;charset=utf-8\r\n\n";
343 // parse query string
344 std::vector<std::string> parts = split(queryString,'&');
345 std::string searchFor;
346 int num=1,page=0;
347 for (std::vector<std::string>::const_iterator it=parts.begin();it!=parts.end();++it)
348 {
349 std::vector<std::string> kv = split(*it,'=');
350 if (kv.size()==2)
351 {
352 std::string val = uriDecode(kv[1]);
353 if (kv[0]=="q") searchFor = val;
354 else if (kv[0]=="n") num = fromString<int>(val);
355 else if (kv[0]=="p") page = fromString<int>(val);
356 else if (kv[0]=="cb") callback = val;
357 }
358 }
359
360 std::string indexDir = "doxysearch.db";
361
362 if (queryString=="test") // user test
363 {
364 bool dbOk = dirExists(indexDir);
365 if (dbOk)
366 {
367 std::cout << "Test successful.";
368 }
369 else
370 {
371 std::cout << "Test failed: cannot find search index " << indexDir;
372 }
373 exit(0);
374 }
375
376 // create query
377 Xapian::Database db(indexDir);
378 Xapian::Enquire enquire(db);
379
380 std::vector<std::string> words = split(searchFor,' ');
381 Xapian::QueryParser parser;
382 parser.set_database(db);
383 parser.set_default_op(Xapian::Query::OP_AND);
384 parser.set_stemming_strategy(Xapian::QueryParser::STEM_ALL);
385 Xapian::termcount max_expansion=100;
386 #if (XAPIAN_MAJOR_VERSION==1) && (XAPIAN_MINOR_VERSION==2)
387 parser.set_max_wildcard_expansion(max_expansion);
388 #else
389 parser.set_max_expansion(max_expansion,Xapian::Query::WILDCARD_LIMIT_MOST_FREQUENT);
390 #endif
391 Xapian::Query query=parser.parse_query(searchFor,
392 Xapian::QueryParser::FLAG_DEFAULT |
393 Xapian::QueryParser::FLAG_WILDCARD |
394 Xapian::QueryParser::FLAG_PHRASE |
395 Xapian::QueryParser::FLAG_PARTIAL
396 );
397 enquire.set_query(query);
398
399 // get results
400 Xapian::MSet matches = enquire.get_mset(page*num,num);
401 unsigned int hits = matches.get_matches_estimated();
402 unsigned int offset = page*num;
403 unsigned int pages = num>0 ? (hits+num-1)/num : 0;
404 if (offset>hits) offset=hits;
405 if (offset+num>hits) num=hits-offset;
406
407 // write results as JSONP
408 std::cout << callback.c_str() << "(";
409 std::cout << "{" << std::endl
410 << " \"hits\":" << hits << "," << std::endl
411 << " \"first\":" << offset << "," << std::endl
412 << " \"count\":" << num << "," << std::endl
413 << " \"page\":" << page << "," << std::endl
414 << " \"pages\":" << pages << "," << std::endl
415 << " \"query\": \"" << escapeString(searchFor) << "\"," << std::endl
416 << " \"items\":[" << std::endl;
417 // foreach search result
418 unsigned int o = offset;
419 for (Xapian::MSetIterator i = matches.begin(); i != matches.end(); ++i,++o)
420 {
421 std::vector<Fragment> hl;
422 Xapian::Document doc = i.get_document();
423 highlighter(doc.get_value(FIELD_DOC),words,hl);
424 std::cout << " {\"type\": \"" << doc.get_value(FIELD_TYPE) << "\"," << std::endl
425 << " \"name\": \"" << doc.get_value(FIELD_NAME) << escapeString(doc.get_value(FIELD_ARGS)) << "\"," << std::endl
426 << " \"tag\": \"" << doc.get_value(FIELD_TAG) << "\"," << std::endl
427 << " \"url\": \"" << doc.get_value(FIELD_URL) << "\"," << std::endl;
428 std::cout << " \"fragments\":[" << std::endl;
429 int c=0;
430 bool first=true;
431 for (std::vector<Fragment>::const_iterator it = hl.begin();it!=hl.end() && c<3;++it,++c)
432 {
433 if (!first) std::cout << "," << std::endl;
434 std::cout << " \"" << escapeString((*it).text) << "\"";
435 first=false;
436 }
437 if (!first) std::cout << std::endl;
438 std::cout << " ]" << std::endl;
439 std::cout << " }";
440 if (o<offset+num-1) std::cout << ",";
441 std::cout << std::endl;
442 }
443 std::cout << " ]" << std::endl << "})" << std::endl;
444 }
445 catch (const Xapian::Error &e) // Xapian exception
446 {
447 showError(callback,e.get_description());
448 }
449 catch (...) // Any other exception
450 {
451 showError(callback,"Unknown Exception!");
452 exit(1);
453 }
454 return 0;
455 }