Debian Codegraph @

addon/doxysearch/doxyindexer.cpp

raw
1 /******************************************************************************
2 *
3 * Copyright (C) 1997-2015 by Dimitri van Heesch.
4 *
5 * Permission to use, copy, modify, and distribute this software and its
6 * documentation under the terms of the GNU General Public License is hereby
7 * granted. No representations are made about the suitability of this software
8 * for any purpose. It is provided "as is" without express or implied warranty.
9 * See the GNU General Public License for more details.
10 *
11 * Documents produced by Doxygen are derivative works derived from the
12 * input used in their production; they are not affected by this license.
13 *
14 */
15
16 // STL includes
17 #include <cstdio>
18 #include <cstdlib>
19 #include <iostream>
20 #include <string>
21 #include <sstream>
22 #include <fstream>
23 #include <iterator>
24 #include <regex>
25 #include <filesystem>
26 #include <unordered_map>
27
28
29 // Xapian include
30 #include <xapian.h>
31
32 #include "version.h"
33 #include "xml.h"
34
35 #define MAX_TERM_LENGTH 245
36
37 #if defined(_WIN32) && !defined(__CYGWIN__)
38 static char pathSep = '\\';
39 #else
40 static char pathSep = '/';
41 #endif
42
43 static void safeAddTerm(const std::string &term,Xapian::Document &doc,int wfd)
44 {
45 if (term.length()<=MAX_TERM_LENGTH) doc.add_term(term,wfd);
46 }
47
48 /** trims \a whitespace characters from the start and end of string \a str. */
49 static std::string trim(const std::string& str,
50 const std::string& whitespace = " \t")
51 {
52 size_t strBegin = str.find_first_not_of(whitespace);
53 if (strBegin == std::string::npos)
54 return ""; // no content
55
56 size_t strEnd = str.find_last_not_of(whitespace);
57 size_t strRange = strEnd - strBegin + 1;
58
59 return str.substr(strBegin, strRange);
60 }
61
62 /** trims \a whitespace from start and end and replace occurrences of
63 * \a whitespace with \a fill.
64 */
65 static std::string reduce(const std::string& str,
66 const std::string& fill = " ",
67 const std::string& whitespace = " \t")
68 {
69 // trim first
70 std::string result = trim(str, whitespace);
71
72 // replace sub ranges
73 size_t beginSpace = result.find_first_of(whitespace);
74 while (beginSpace != std::string::npos)
75 {
76 size_t endSpace = result.find_first_not_of(whitespace, beginSpace);
77 size_t range = endSpace - beginSpace;
78
79 result.replace(beginSpace, range, fill);
80
81 size_t newStart = beginSpace + fill.length();
82 beginSpace = result.find_first_of(whitespace, newStart);
83 }
84
85 return result;
86 }
87
88 /** Adds all words in \a s to document \a doc with weight \a wfd */
89 static void addWords(const std::string &s,Xapian::Document &doc,int wfd)
90 {
91 std::istringstream iss(s);
92 std::istream_iterator<std::string> begin(iss),end,it;
93 for (it=begin;it!=end;++it)
94 {
95 const std::string word = *it;
96 const std::string lword = Xapian::Unicode::tolower(word);
97 safeAddTerm(word,doc,wfd);
98 if (lword!=word)
99 {
100 safeAddTerm(lword,doc,wfd);
101 }
102 }
103 }
104
105 /** Adds all identifiers in \a s to document \a doc with weight \a wfd */
106 static void addIdentifiers(const std::string &s,Xapian::Document &doc,int wfd)
107 {
108 static const std::regex id_re("[A-Z_a-z][A-Z_a-z0-9]*");
109 auto id_begin = std::sregex_iterator(s.begin(), s.end(), id_re);
110 auto id_end = std::sregex_iterator();
111
112 for (auto i = id_begin; i!=id_end; ++i)
113 {
114 std::smatch match = *i;
115 safeAddTerm(match.str(),doc,wfd);
116 }
117 }
118
119 /** Replaces all occurrences of \a old with \a repl in string \a str */
120 static void replace_all(std::string& str, const std::string& old, const std::string& repl)
121 {
122 size_t pos = 0;
123 while ((pos = str.find(old, pos)) != std::string::npos)
124 {
125 str.replace(pos, old.length(), repl);
126 pos += repl.length();
127 }
128 }
129
130 /** Replaces all XML entities in \a s with their unescaped representation */
131 static std::string unescapeXmlEntities(const std::string &s)
132 {
133 std::string result=s;
134 replace_all(result,"&gt;",">");
135 replace_all(result,"&lt;","<");
136 replace_all(result,"&apos;","'");
137 replace_all(result,"&quot;","\"");
138 replace_all(result,"&amp;","&");
139 return result;
140 }
141
142 /** This class is a wrapper around SAX style XML parser, which
143 * parses the file without first building a DOM tree in memory.
144 */
145 class XMLContentHandler
146 {
147 public:
148 /** Handler for parsing XML data */
149 XMLContentHandler(const std::string &path)
150 : m_db(path+"doxysearch.db",Xapian::DB_CREATE_OR_OVERWRITE),
151 m_stemmer("english")
152 {
153 m_curFieldName = UnknownField;
154 m_indexer.set_stemmer(m_stemmer);
155 m_indexer.set_document(m_doc);
156 }
157
158 /** Free data handler */
159 ~XMLContentHandler()
160 {
161 m_db.commit();
162 }
163
164 enum FieldNames
165 {
166 UnknownField = 0,
167 TypeField = 1,
168 NameField = 2,
169 ArgsField = 3,
170 TagField = 4,
171 UrlField = 5,
172 KeywordField = 6,
173 TextField = 7
174 };
175
176 /** Handler for a start tag. Called for `<doc>` and `<field>` tags */
177 void startElement(const std::string &name, const XMLHandlers::Attributes &attrib)
178 {
179 m_data="";
180
181 if (name != "field") return;
182
183 static const std::unordered_map<std::string, FieldNames> fieldMap{
184 { "type", TypeField },
185 { "name", NameField },
186 { "args", ArgsField },
187 { "tag", TagField },
188 { "url", UrlField },
189 { "keywords", KeywordField },
190 { "text", TextField }
191 };
192 std::string fieldName = XMLHandlers::value(attrib, "name");
193 auto it = fieldMap.find(fieldName);
194 m_curFieldName = (it != fieldMap.end()) ? it->second : UnknownField;
195 }
196
197 /** Handler for an end tag. Called for `</doc>` and `</field>` tags */
198 void endElement(const std::string &name)
199 {
200 if (name=="doc") // </doc>
201 {
202 std::string term = m_doc.get_value(NameField);
203 std::string partTerm;
204 size_t pos = term.rfind("::");
205 if (pos!=std::string::npos)
206 {
207 partTerm = term.substr(pos+2);
208 }
209 if (m_doc.get_value(TypeField)=="class" ||
210 m_doc.get_value(TypeField)=="file" ||
211 m_doc.get_value(TypeField)=="namespace") // containers get highest prio
212 {
213 safeAddTerm(term,m_doc,1000);
214 if (!partTerm.empty())
215 {
216 safeAddTerm(partTerm,m_doc,500);
217 }
218 }
219 else // members and others get lower prio
220 {
221 safeAddTerm(m_doc.get_value(NameField),m_doc,100);
222 if (!partTerm.empty())
223 {
224 safeAddTerm(partTerm,m_doc,50);
225 }
226 }
227 m_db.add_document(m_doc);
228 m_doc.clear_values();
229 m_doc.clear_terms();
230 }
231 else if (name=="field" && m_curFieldName!=UnknownField) // </field>
232 {
233 // strip whitespace from m_data
234 m_data = reduce(m_data);
235 // replace XML entities
236 m_data = unescapeXmlEntities(m_data);
237 // add data to the document
238 m_doc.add_value(m_curFieldName,m_data);
239 switch (m_curFieldName)
240 {
241 case TypeField:
242 case NameField:
243 case TagField:
244 case UrlField:
245 // meta data that is not searchable
246 break;
247 case KeywordField:
248 addWords(m_data,m_doc,50);
249 break;
250 case ArgsField:
251 addIdentifiers(m_data,m_doc,10);
252 break;
253 case TextField:
254 addWords(m_data,m_doc,2);
255 break;
256 default:
257 break;
258 }
259 m_data="";
260 m_curFieldName=UnknownField;
261 }
262 // reset m_data
263 }
264
265 /** Handler for inline text */
266 void characters(const std::string& ch)
267 {
268 m_data += ch;
269 }
270
271 void error(const std::string &fileName,int lineNr,const std::string &msg)
272 {
273 std::cerr << "Fatal error at " << fileName << ":" << lineNr << ": " << msg << std::endl;
274 }
275
276 private:
277
278 // internal state
279 Xapian::WritableDatabase m_db;
280 Xapian::Document m_doc;
281 Xapian::TermGenerator m_indexer;
282 Xapian::Stem m_stemmer;
283 std::string m_data;
284 FieldNames m_curFieldName;
285 };
286
287 static void usage(const char *name, int exitVal = 1)
288 {
289 std::cerr << "Usage: " << name << " [-o output_dir] searchdata.xml [searchdata2.xml ...]" << std::endl;
290 exit(exitVal);
291 }
292
293 // return the contents of a file as a string
294 inline std::string fileToString(const std::string &fileName)
295 {
296 std::ifstream t(fileName);
297 std::string result;
298 t.seekg(0, std::ios::end);
299 auto size = t.tellg();
300 if (size < 0) size = 0;
301 result.reserve(static_cast<std::size_t>(size));
302 t.seekg(0, std::ios::beg);
303 result.assign(std::istreambuf_iterator<char>(t),
304 std::istreambuf_iterator<char>());
305 return result;
306 }
307
308 bool dirExists(const char *path)
309 {
310 std::error_code ec;
311 return std::filesystem::is_directory(path, ec);
312 }
313
314 /** main function to index data */
315 int main(int argc,const char **argv)
316 {
317 if (argc<2)
318 {
319 usage(argv[0]);
320 }
321 std::string outputDir;
322 for (int i=1;i<argc;i++)
323 {
324 const std::string arg{ argv[i] };
325 if (arg == "-o")
326 {
327 if (i>=argc-1)
328 {
329 std::cerr << "Error: missing parameter for -o option" << std::endl;
330 usage(argv[0]);
331 }
332 else
333 {
334 i++;
335 outputDir=argv[i];
336 if (!dirExists(outputDir.c_str()))
337 {
338 std::cerr << "Error: specified output directory does not exist!" << std::endl;
339 usage(argv[0]);
340 }
341 }
342 }
343 else if (arg == "-h" || arg == "--help")
344 {
345 usage(argv[0],0);
346 }
347 else if (arg == "-v" || arg == "--version")
348 {
349 std::cerr << argv[0] << " version: " << getFullVersion() << std::endl;
350 exit(0);
351 }
352 }
353
354 try
355 {
356 if (!outputDir.empty() && outputDir.at(outputDir.length()-1)!=pathSep)
357 {
358 outputDir+=pathSep;
359 }
360 XMLContentHandler contentHandler(outputDir);
361 XMLHandlers handlers;
362 handlers.startElement = [&contentHandler](const std::string &name,const XMLHandlers::Attributes &attrs) { contentHandler.startElement(name,attrs); };
363 handlers.endElement = [&contentHandler](const std::string &name) { contentHandler.endElement(name); };
364 handlers.characters = [&contentHandler](const std::string &chars) { contentHandler.characters(chars); };
365 handlers.error = [&contentHandler](const std::string &fileName,int lineNr,const std::string &msg) { contentHandler.error(fileName,lineNr,msg); };
366 for (int i=1;i<argc;i++)
367 {
368 if (std::string(argv[i])=="-o")
369 {
370 i++;
371 }
372 else
373 {
374 std::cout << "Processing " << argv[i] << "..." << std::endl;
375 std::string inputStr = fileToString(argv[i]);
376 XMLParser parser(handlers);
377 parser.parse(argv[i],inputStr.c_str(),false,[](){},[](){});
378 }
379 }
380 }
381 catch(const Xapian::Error &e)
382 {
383 std::cerr << "Caught exception: " << e.get_description() << std::endl;
384 }
385 catch(...)
386 {
387 std::cerr << "Caught an unknown exception" << std::endl;
388 }
389
390 return 0;
391 }