Debian Codegraph @

src/backend/tsearch/wparser_def.c

raw
1 /*-------------------------------------------------------------------------
2 *
3 * wparser_def.c
4 * Default text search parser
5 *
6 * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
7 *
8 *
9 * IDENTIFICATION
10 * src/backend/tsearch/wparser_def.c
11 *
12 *-------------------------------------------------------------------------
13 */
14
15 #include "postgres.h"
16
17 #include <limits.h>
18 #include <wctype.h>
19
20 #include "commands/defrem.h"
21 #include "mb/pg_wchar.h"
22 #include "miscadmin.h"
23 #include "tsearch/ts_public.h"
24 #include "tsearch/ts_type.h"
25 #include "tsearch/ts_utils.h"
26 #include "utils/builtins.h"
27 #include "utils/pg_locale.h"
28
29
30 /* Define me to enable tracing of parser behavior */
31 /* #define WPARSER_TRACE */
32
33
34 /* Output token categories */
35
36 #define ASCIIWORD 1
37 #define WORD_T 2
38 #define NUMWORD 3
39 #define EMAIL 4
40 #define URL_T 5
41 #define HOST 6
42 #define SCIENTIFIC 7
43 #define VERSIONNUMBER 8
44 #define NUMPARTHWORD 9
45 #define PARTHWORD 10
46 #define ASCIIPARTHWORD 11
47 #define SPACE 12
48 #define TAG_T 13
49 #define PROTOCOL 14
50 #define NUMHWORD 15
51 #define ASCIIHWORD 16
52 #define HWORD 17
53 #define URLPATH 18
54 #define FILEPATH 19
55 #define DECIMAL_T 20
56 #define SIGNEDINT 21
57 #define UNSIGNEDINT 22
58 #define XMLENTITY 23
59
60 #define LASTNUM 23
61
62 static const char *const tok_alias[] = {
63 "",
64 "asciiword",
65 "word",
66 "numword",
67 "email",
68 "url",
69 "host",
70 "sfloat",
71 "version",
72 "hword_numpart",
73 "hword_part",
74 "hword_asciipart",
75 "blank",
76 "tag",
77 "protocol",
78 "numhword",
79 "asciihword",
80 "hword",
81 "url_path",
82 "file",
83 "float",
84 "int",
85 "uint",
86 "entity"
87 };
88
89 static const char *const lex_descr[] = {
90 "",
91 "Word, all ASCII",
92 "Word, all letters",
93 "Word, letters and digits",
94 "Email address",
95 "URL",
96 "Host",
97 "Scientific notation",
98 "Version number",
99 "Hyphenated word part, letters and digits",
100 "Hyphenated word part, all letters",
101 "Hyphenated word part, all ASCII",
102 "Space symbols",
103 "XML tag",
104 "Protocol head",
105 "Hyphenated word, letters and digits",
106 "Hyphenated word, all ASCII",
107 "Hyphenated word, all letters",
108 "URL path",
109 "File or path name",
110 "Decimal notation",
111 "Signed integer",
112 "Unsigned integer",
113 "XML entity"
114 };
115
116
117 /* Parser states */
118
119 typedef enum
120 {
121 TPS_Base = 0,
122 TPS_InNumWord,
123 TPS_InAsciiWord,
124 TPS_InWord,
125 TPS_InUnsignedInt,
126 TPS_InSignedIntFirst,
127 TPS_InSignedInt,
128 TPS_InSpace,
129 TPS_InUDecimalFirst,
130 TPS_InUDecimal,
131 TPS_InDecimalFirst,
132 TPS_InDecimal,
133 TPS_InVerVersion,
134 TPS_InSVerVersion,
135 TPS_InVersionFirst,
136 TPS_InVersion,
137 TPS_InMantissaFirst,
138 TPS_InMantissaSign,
139 TPS_InMantissa,
140 TPS_InXMLEntityFirst,
141 TPS_InXMLEntity,
142 TPS_InXMLEntityNumFirst,
143 TPS_InXMLEntityNum,
144 TPS_InXMLEntityHexNumFirst,
145 TPS_InXMLEntityHexNum,
146 TPS_InXMLEntityEnd,
147 TPS_InTagFirst,
148 TPS_InXMLBegin,
149 TPS_InTagCloseFirst,
150 TPS_InTagName,
151 TPS_InTagBeginEnd,
152 TPS_InTag,
153 TPS_InTagEscapeK,
154 TPS_InTagEscapeKK,
155 TPS_InTagBackSleshed,
156 TPS_InTagEnd,
157 TPS_InCommentFirst,
158 TPS_InCommentLast,
159 TPS_InComment,
160 TPS_InCloseCommentFirst,
161 TPS_InCloseCommentLast,
162 TPS_InCommentEnd,
163 TPS_InHostFirstDomain,
164 TPS_InHostDomainSecond,
165 TPS_InHostDomain,
166 TPS_InPortFirst,
167 TPS_InPort,
168 TPS_InHostFirstAN,
169 TPS_InHost,
170 TPS_InEmail,
171 TPS_InFileFirst,
172 TPS_InFileTwiddle,
173 TPS_InPathFirst,
174 TPS_InPathFirstFirst,
175 TPS_InPathSecond,
176 TPS_InFile,
177 TPS_InFileNext,
178 TPS_InURLPathFirst,
179 TPS_InURLPathStart,
180 TPS_InURLPath,
181 TPS_InFURL,
182 TPS_InProtocolFirst,
183 TPS_InProtocolSecond,
184 TPS_InProtocolEnd,
185 TPS_InHyphenAsciiWordFirst,
186 TPS_InHyphenAsciiWord,
187 TPS_InHyphenWordFirst,
188 TPS_InHyphenWord,
189 TPS_InHyphenNumWordFirst,
190 TPS_InHyphenNumWord,
191 TPS_InHyphenDigitLookahead,
192 TPS_InParseHyphen,
193 TPS_InParseHyphenHyphen,
194 TPS_InHyphenWordPart,
195 TPS_InHyphenAsciiWordPart,
196 TPS_InHyphenNumWordPart,
197 TPS_InHyphenUnsignedInt,
198 TPS_Null /* last state (fake value) */
199 } TParserState;
200
201 /* forward declaration */
202 struct TParser;
203
204 typedef int (*TParserCharTest) (struct TParser *); /* any p_is* functions
205 * except p_iseq */
206 typedef void (*TParserSpecial) (struct TParser *); /* special handler for
207 * special cases... */
208
209 typedef struct
210 {
211 TParserCharTest isclass;
212 char c;
213 uint16 flags;
214 TParserState tostate;
215 int type;
216 TParserSpecial special;
217 } TParserStateActionItem;
218
219 /* Flag bits in TParserStateActionItem.flags */
220 #define A_NEXT 0x0000
221 #define A_BINGO 0x0001
222 #define A_POP 0x0002
223 #define A_PUSH 0x0004
224 #define A_RERUN 0x0008
225 #define A_CLEAR 0x0010
226 #define A_MERGE 0x0020
227 #define A_CLRALL 0x0040
228
229 typedef struct TParserPosition
230 {
231 int posbyte; /* position of parser in bytes */
232 int poschar; /* position of parser in characters */
233 int charlen; /* length of current char */
234 int lenbytetoken; /* length of token-so-far in bytes */
235 int lenchartoken; /* and in chars */
236 TParserState state;
237 struct TParserPosition *prev;
238 const TParserStateActionItem *pushedAtAction;
239 } TParserPosition;
240
241 typedef struct TParser
242 {
243 /* string and position information */
244 char *str; /* multibyte string */
245 int lenstr; /* length of mbstring */
246 wchar_t *wstr; /* wide character string */
247 pg_wchar *pgwstr; /* wide character string for C-locale */
248 bool usewide;
249
250 /* State of parse */
251 int charmaxlen;
252 TParserPosition *state;
253 bool ignore;
254 bool wanthost;
255
256 /* silly char */
257 char c;
258
259 /* out */
260 char *token;
261 int lenbytetoken;
262 int lenchartoken;
263 int type;
264 } TParser;
265
266
267 /* forward decls here */
268 static bool TParserGet(TParser *prs);
269
270
271 static TParserPosition *
272 newTParserPosition(TParserPosition *prev)
273 {
274 TParserPosition *res = (TParserPosition *) palloc(sizeof(TParserPosition));
275
276 if (prev)
277 memcpy(res, prev, sizeof(TParserPosition));
278 else
279 memset(res, 0, sizeof(TParserPosition));
280
281 res->prev = prev;
282
283 res->pushedAtAction = NULL;
284
285 return res;
286 }
287
288 static TParser *
289 TParserInit(char *str, int len)
290 {
291 TParser *prs = (TParser *) palloc0(sizeof(TParser));
292
293 prs->charmaxlen = pg_database_encoding_max_length();
294 prs->str = str;
295 prs->lenstr = len;
296
297 /*
298 * Use wide char code only when max encoding length > 1.
299 */
300 if (prs->charmaxlen > 1)
301 {
302 pg_locale_t mylocale = 0; /* TODO */
303
304 prs->usewide = true;
305 if (database_ctype_is_c)
306 {
307 /*
308 * char2wchar doesn't work for C-locale and sizeof(pg_wchar) could
309 * be different from sizeof(wchar_t)
310 */
311 prs->pgwstr = (pg_wchar *) palloc(sizeof(pg_wchar) * (prs->lenstr + 1));
312 pg_mb2wchar_with_len(prs->str, prs->pgwstr, prs->lenstr);
313 }
314 else
315 {
316 prs->wstr = (wchar_t *) palloc(sizeof(wchar_t) * (prs->lenstr + 1));
317 char2wchar(prs->wstr, prs->lenstr + 1, prs->str, prs->lenstr,
318 mylocale);
319 }
320 }
321 else
322 prs->usewide = false;
323
324 prs->state = newTParserPosition(NULL);
325 prs->state->state = TPS_Base;
326
327 #ifdef WPARSER_TRACE
328 fprintf(stderr, "parsing \"%.*s\"\n", len, str);
329 #endif
330
331 return prs;
332 }
333
334 /*
335 * As an alternative to a full TParserInit one can create a
336 * TParserCopy which basically is a regular TParser without a private
337 * copy of the string - instead it uses the one from another TParser.
338 * This is useful because at some places TParsers are created
339 * recursively and the repeated copying around of the strings can
340 * cause major inefficiency if the source string is long.
341 * The new parser starts parsing at the original's current position.
342 *
343 * Obviously one must not close the original TParser before the copy.
344 */
345 static TParser *
346 TParserCopyInit(const TParser *orig)
347 {
348 TParser *prs = (TParser *) palloc0(sizeof(TParser));
349
350 prs->charmaxlen = orig->charmaxlen;
351 prs->str = orig->str + orig->state->posbyte;
352 prs->lenstr = orig->lenstr - orig->state->posbyte;
353 prs->usewide = orig->usewide;
354
355 if (orig->pgwstr)
356 prs->pgwstr = orig->pgwstr + orig->state->poschar;
357 if (orig->wstr)
358 prs->wstr = orig->wstr + orig->state->poschar;
359
360 prs->state = newTParserPosition(NULL);
361 prs->state->state = TPS_Base;
362
363 #ifdef WPARSER_TRACE
364 fprintf(stderr, "parsing copy of \"%.*s\"\n", prs->lenstr, prs->str);
365 #endif
366
367 return prs;
368 }
369
370
371 static void
372 TParserClose(TParser *prs)
373 {
374 while (prs->state)
375 {
376 TParserPosition *ptr = prs->state->prev;
377
378 pfree(prs->state);
379 prs->state = ptr;
380 }
381
382 if (prs->wstr)
383 pfree(prs->wstr);
384 if (prs->pgwstr)
385 pfree(prs->pgwstr);
386
387 #ifdef WPARSER_TRACE
388 fprintf(stderr, "closing parser\n");
389 #endif
390 pfree(prs);
391 }
392
393 /*
394 * Close a parser created with TParserCopyInit
395 */
396 static void
397 TParserCopyClose(TParser *prs)
398 {
399 while (prs->state)
400 {
401 TParserPosition *ptr = prs->state->prev;
402
403 pfree(prs->state);
404 prs->state = ptr;
405 }
406
407 #ifdef WPARSER_TRACE
408 fprintf(stderr, "closing parser copy\n");
409 #endif
410 pfree(prs);
411 }
412
413
414 /*
415 * Character-type support functions, equivalent to is* macros, but
416 * working with any possible encodings and locales. Notes:
417 * - with multibyte encoding and C-locale isw* function may fail
418 * or give wrong result.
419 * - multibyte encoding and C-locale often are used for
420 * Asian languages.
421 * - if locale is C then we use pgwstr instead of wstr.
422 */
423
424 #define p_iswhat(type, nonascii) \
425 \
426 static int \
427 p_is##type(TParser *prs) \
428 { \
429 Assert(prs->state); \
430 if (prs->usewide) \
431 { \
432 if (prs->pgwstr) \
433 { \
434 unsigned int c = *(prs->pgwstr + prs->state->poschar); \
435 if (c > 0x7f) \
436 return nonascii; \
437 return is##type(c); \
438 } \
439 return isw##type(*(prs->wstr + prs->state->poschar)); \
440 } \
441 return is##type(*(unsigned char *) (prs->str + prs->state->posbyte)); \
442 } \
443 \
444 static int \
445 p_isnot##type(TParser *prs) \
446 { \
447 return !p_is##type(prs); \
448 }
449
450 /*
451 * In C locale with a multibyte encoding, any non-ASCII symbol is considered
452 * an alpha character, but not a member of other char classes.
453 */
454 p_iswhat(alnum, 1)
455 p_iswhat(alpha, 1)
456 p_iswhat(digit, 0)
457 p_iswhat(lower, 0)
458 p_iswhat(print, 0)
459 p_iswhat(punct, 0)
460 p_iswhat(space, 0)
461 p_iswhat(upper, 0)
462 p_iswhat(xdigit, 0)
463
464 /* p_iseq should be used only for ascii symbols */
465
466 static int
467 p_iseq(TParser *prs, char c)
468 {
469 Assert(prs->state);
470 return ((prs->state->charlen == 1 && *(prs->str + prs->state->posbyte) == c)) ? 1 : 0;
471 }
472
473 static int
474 p_isEOF(TParser *prs)
475 {
476 Assert(prs->state);
477 return (prs->state->posbyte == prs->lenstr || prs->state->charlen == 0) ? 1 : 0;
478 }
479
480 static int
481 p_iseqC(TParser *prs)
482 {
483 return p_iseq(prs, prs->c);
484 }
485
486 static int
487 p_isneC(TParser *prs)
488 {
489 return !p_iseq(prs, prs->c);
490 }
491
492 static int
493 p_isascii(TParser *prs)
494 {
495 return (prs->state->charlen == 1 && isascii((unsigned char) *(prs->str + prs->state->posbyte))) ? 1 : 0;
496 }
497
498 static int
499 p_isasclet(TParser *prs)
500 {
501 return (p_isascii(prs) && p_isalpha(prs)) ? 1 : 0;
502 }
503
504 static int
505 p_isurlchar(TParser *prs)
506 {
507 char ch;
508
509 /* no non-ASCII need apply */
510 if (prs->state->charlen != 1)
511 return 0;
512 ch = *(prs->str + prs->state->posbyte);
513 /* no spaces or control characters */
514 if (ch <= 0x20 || ch >= 0x7F)
515 return 0;
516 /* reject characters disallowed by RFC 3986 */
517 switch (ch)
518 {
519 case '"':
520 case '<':
521 case '>':
522 case '\\':
523 case '^':
524 case '`':
525 case '{':
526 case '|':
527 case '}':
528 return 0;
529 }
530 return 1;
531 }
532
533
534 /* deliberately suppress unused-function complaints for the above */
535 void _make_compiler_happy(void);
536 void
537 _make_compiler_happy(void)
538 {
539 p_isalnum(NULL);
540 p_isnotalnum(NULL);
541 p_isalpha(NULL);
542 p_isnotalpha(NULL);
543 p_isdigit(NULL);
544 p_isnotdigit(NULL);
545 p_islower(NULL);
546 p_isnotlower(NULL);
547 p_isprint(NULL);
548 p_isnotprint(NULL);
549 p_ispunct(NULL);
550 p_isnotpunct(NULL);
551 p_isspace(NULL);
552 p_isnotspace(NULL);
553 p_isupper(NULL);
554 p_isnotupper(NULL);
555 p_isxdigit(NULL);
556 p_isnotxdigit(NULL);
557 p_isEOF(NULL);
558 p_iseqC(NULL);
559 p_isneC(NULL);
560 }
561
562
563 static void
564 SpecialTags(TParser *prs)
565 {
566 switch (prs->state->lenchartoken)
567 {
568 case 8: /* </script */
569 if (pg_strncasecmp(prs->token, "</script", 8) == 0)
570 prs->ignore = false;
571 break;
572 case 7: /* <script || </style */
573 if (pg_strncasecmp(prs->token, "</style", 7) == 0)
574 prs->ignore = false;
575 else if (pg_strncasecmp(prs->token, "<script", 7) == 0)
576 prs->ignore = true;
577 break;
578 case 6: /* <style */
579 if (pg_strncasecmp(prs->token, "<style", 6) == 0)
580 prs->ignore = true;
581 break;
582 default:
583 break;
584 }
585 }
586
587 static void
588 SpecialFURL(TParser *prs)
589 {
590 prs->wanthost = true;
591 prs->state->posbyte -= prs->state->lenbytetoken;
592 prs->state->poschar -= prs->state->lenchartoken;
593 }
594
595 static void
596 SpecialHyphen(TParser *prs)
597 {
598 prs->state->posbyte -= prs->state->lenbytetoken;
599 prs->state->poschar -= prs->state->lenchartoken;
600 }
601
602 static void
603 SpecialVerVersion(TParser *prs)
604 {
605 prs->state->posbyte -= prs->state->lenbytetoken;
606 prs->state->poschar -= prs->state->lenchartoken;
607 prs->state->lenbytetoken = 0;
608 prs->state->lenchartoken = 0;
609 }
610
611 static int
612 p_isstophost(TParser *prs)
613 {
614 if (prs->wanthost)
615 {
616 prs->wanthost = false;
617 return 1;
618 }
619 return 0;
620 }
621
622 static int
623 p_isignore(TParser *prs)
624 {
625 return (prs->ignore) ? 1 : 0;
626 }
627
628 static int
629 p_ishost(TParser *prs)
630 {
631 TParser *tmpprs = TParserCopyInit(prs);
632 int res = 0;
633
634 tmpprs->wanthost = true;
635
636 /*
637 * Check stack depth before recursing. (Since TParserGet() doesn't
638 * normally recurse, we put the cost of checking here not there.)
639 */
640 check_stack_depth();
641
642 if (TParserGet(tmpprs) && tmpprs->type == HOST)
643 {
644 prs->state->posbyte += tmpprs->lenbytetoken;
645 prs->state->poschar += tmpprs->lenchartoken;
646 prs->state->lenbytetoken += tmpprs->lenbytetoken;
647 prs->state->lenchartoken += tmpprs->lenchartoken;
648 prs->state->charlen = tmpprs->state->charlen;
649 res = 1;
650 }
651 TParserCopyClose(tmpprs);
652
653 return res;
654 }
655
656 static int
657 p_isURLPath(TParser *prs)
658 {
659 TParser *tmpprs = TParserCopyInit(prs);
660 int res = 0;
661
662 tmpprs->state = newTParserPosition(tmpprs->state);
663 tmpprs->state->state = TPS_InURLPathFirst;
664
665 /*
666 * Check stack depth before recursing. (Since TParserGet() doesn't
667 * normally recurse, we put the cost of checking here not there.)
668 */
669 check_stack_depth();
670
671 if (TParserGet(tmpprs) && tmpprs->type == URLPATH)
672 {
673 prs->state->posbyte += tmpprs->lenbytetoken;
674 prs->state->poschar += tmpprs->lenchartoken;
675 prs->state->lenbytetoken += tmpprs->lenbytetoken;
676 prs->state->lenchartoken += tmpprs->lenchartoken;
677 prs->state->charlen = tmpprs->state->charlen;
678 res = 1;
679 }
680 TParserCopyClose(tmpprs);
681
682 return res;
683 }
684
685 /*
686 * returns true if current character has zero display length or
687 * it's a special sign in several languages. Such characters
688 * aren't a word-breaker although they aren't an isalpha.
689 * In beginning of word they aren't a part of it.
690 */
691 static int
692 p_isspecial(TParser *prs)
693 {
694 /*
695 * pg_dsplen could return -1 which means error or control character
696 */
697 if (pg_dsplen(prs->str + prs->state->posbyte) == 0)
698 return 1;
699
700 /*
701 * Unicode Characters in the 'Mark, Spacing Combining' Category That
702 * characters are not alpha although they are not breakers of word too.
703 * Check that only in utf encoding, because other encodings aren't
704 * supported by postgres or even exists.
705 */
706 if (GetDatabaseEncoding() == PG_UTF8 && prs->usewide)
707 {
708 static const pg_wchar strange_letter[] = {
709 /*
710 * use binary search, so elements should be ordered
711 */
712 0x0903, /* DEVANAGARI SIGN VISARGA */
713 0x093E, /* DEVANAGARI VOWEL SIGN AA */
714 0x093F, /* DEVANAGARI VOWEL SIGN I */
715 0x0940, /* DEVANAGARI VOWEL SIGN II */
716 0x0949, /* DEVANAGARI VOWEL SIGN CANDRA O */
717 0x094A, /* DEVANAGARI VOWEL SIGN SHORT O */
718 0x094B, /* DEVANAGARI VOWEL SIGN O */
719 0x094C, /* DEVANAGARI VOWEL SIGN AU */
720 0x0982, /* BENGALI SIGN ANUSVARA */
721 0x0983, /* BENGALI SIGN VISARGA */
722 0x09BE, /* BENGALI VOWEL SIGN AA */
723 0x09BF, /* BENGALI VOWEL SIGN I */
724 0x09C0, /* BENGALI VOWEL SIGN II */
725 0x09C7, /* BENGALI VOWEL SIGN E */
726 0x09C8, /* BENGALI VOWEL SIGN AI */
727 0x09CB, /* BENGALI VOWEL SIGN O */
728 0x09CC, /* BENGALI VOWEL SIGN AU */
729 0x09D7, /* BENGALI AU LENGTH MARK */
730 0x0A03, /* GURMUKHI SIGN VISARGA */
731 0x0A3E, /* GURMUKHI VOWEL SIGN AA */
732 0x0A3F, /* GURMUKHI VOWEL SIGN I */
733 0x0A40, /* GURMUKHI VOWEL SIGN II */
734 0x0A83, /* GUJARATI SIGN VISARGA */
735 0x0ABE, /* GUJARATI VOWEL SIGN AA */
736 0x0ABF, /* GUJARATI VOWEL SIGN I */
737 0x0AC0, /* GUJARATI VOWEL SIGN II */
738 0x0AC9, /* GUJARATI VOWEL SIGN CANDRA O */
739 0x0ACB, /* GUJARATI VOWEL SIGN O */
740 0x0ACC, /* GUJARATI VOWEL SIGN AU */
741 0x0B02, /* ORIYA SIGN ANUSVARA */
742 0x0B03, /* ORIYA SIGN VISARGA */
743 0x0B3E, /* ORIYA VOWEL SIGN AA */
744 0x0B40, /* ORIYA VOWEL SIGN II */
745 0x0B47, /* ORIYA VOWEL SIGN E */
746 0x0B48, /* ORIYA VOWEL SIGN AI */
747 0x0B4B, /* ORIYA VOWEL SIGN O */
748 0x0B4C, /* ORIYA VOWEL SIGN AU */
749 0x0B57, /* ORIYA AU LENGTH MARK */
750 0x0BBE, /* TAMIL VOWEL SIGN AA */
751 0x0BBF, /* TAMIL VOWEL SIGN I */
752 0x0BC1, /* TAMIL VOWEL SIGN U */
753 0x0BC2, /* TAMIL VOWEL SIGN UU */
754 0x0BC6, /* TAMIL VOWEL SIGN E */
755 0x0BC7, /* TAMIL VOWEL SIGN EE */
756 0x0BC8, /* TAMIL VOWEL SIGN AI */
757 0x0BCA, /* TAMIL VOWEL SIGN O */
758 0x0BCB, /* TAMIL VOWEL SIGN OO */
759 0x0BCC, /* TAMIL VOWEL SIGN AU */
760 0x0BD7, /* TAMIL AU LENGTH MARK */
761 0x0C01, /* TELUGU SIGN CANDRABINDU */
762 0x0C02, /* TELUGU SIGN ANUSVARA */
763 0x0C03, /* TELUGU SIGN VISARGA */
764 0x0C41, /* TELUGU VOWEL SIGN U */
765 0x0C42, /* TELUGU VOWEL SIGN UU */
766 0x0C43, /* TELUGU VOWEL SIGN VOCALIC R */
767 0x0C44, /* TELUGU VOWEL SIGN VOCALIC RR */
768 0x0C82, /* KANNADA SIGN ANUSVARA */
769 0x0C83, /* KANNADA SIGN VISARGA */
770 0x0CBE, /* KANNADA VOWEL SIGN AA */
771 0x0CC0, /* KANNADA VOWEL SIGN II */
772 0x0CC1, /* KANNADA VOWEL SIGN U */
773 0x0CC2, /* KANNADA VOWEL SIGN UU */
774 0x0CC3, /* KANNADA VOWEL SIGN VOCALIC R */
775 0x0CC4, /* KANNADA VOWEL SIGN VOCALIC RR */
776 0x0CC7, /* KANNADA VOWEL SIGN EE */
777 0x0CC8, /* KANNADA VOWEL SIGN AI */
778 0x0CCA, /* KANNADA VOWEL SIGN O */
779 0x0CCB, /* KANNADA VOWEL SIGN OO */
780 0x0CD5, /* KANNADA LENGTH MARK */
781 0x0CD6, /* KANNADA AI LENGTH MARK */
782 0x0D02, /* MALAYALAM SIGN ANUSVARA */
783 0x0D03, /* MALAYALAM SIGN VISARGA */
784 0x0D3E, /* MALAYALAM VOWEL SIGN AA */
785 0x0D3F, /* MALAYALAM VOWEL SIGN I */
786 0x0D40, /* MALAYALAM VOWEL SIGN II */
787 0x0D46, /* MALAYALAM VOWEL SIGN E */
788 0x0D47, /* MALAYALAM VOWEL SIGN EE */
789 0x0D48, /* MALAYALAM VOWEL SIGN AI */
790 0x0D4A, /* MALAYALAM VOWEL SIGN O */
791 0x0D4B, /* MALAYALAM VOWEL SIGN OO */
792 0x0D4C, /* MALAYALAM VOWEL SIGN AU */
793 0x0D57, /* MALAYALAM AU LENGTH MARK */
794 0x0D82, /* SINHALA SIGN ANUSVARAYA */
795 0x0D83, /* SINHALA SIGN VISARGAYA */
796 0x0DCF, /* SINHALA VOWEL SIGN AELA-PILLA */
797 0x0DD0, /* SINHALA VOWEL SIGN KETTI AEDA-PILLA */
798 0x0DD1, /* SINHALA VOWEL SIGN DIGA AEDA-PILLA */
799 0x0DD8, /* SINHALA VOWEL SIGN GAETTA-PILLA */
800 0x0DD9, /* SINHALA VOWEL SIGN KOMBUVA */
801 0x0DDA, /* SINHALA VOWEL SIGN DIGA KOMBUVA */
802 0x0DDB, /* SINHALA VOWEL SIGN KOMBU DEKA */
803 0x0DDC, /* SINHALA VOWEL SIGN KOMBUVA HAA AELA-PILLA */
804 0x0DDD, /* SINHALA VOWEL SIGN KOMBUVA HAA DIGA
805 * AELA-PILLA */
806 0x0DDE, /* SINHALA VOWEL SIGN KOMBUVA HAA GAYANUKITTA */
807 0x0DDF, /* SINHALA VOWEL SIGN GAYANUKITTA */
808 0x0DF2, /* SINHALA VOWEL SIGN DIGA GAETTA-PILLA */
809 0x0DF3, /* SINHALA VOWEL SIGN DIGA GAYANUKITTA */
810 0x0F3E, /* TIBETAN SIGN YAR TSHES */
811 0x0F3F, /* TIBETAN SIGN MAR TSHES */
812 0x0F7F, /* TIBETAN SIGN RNAM BCAD */
813 0x102B, /* MYANMAR VOWEL SIGN TALL AA */
814 0x102C, /* MYANMAR VOWEL SIGN AA */
815 0x1031, /* MYANMAR VOWEL SIGN E */
816 0x1038, /* MYANMAR SIGN VISARGA */
817 0x103B, /* MYANMAR CONSONANT SIGN MEDIAL YA */
818 0x103C, /* MYANMAR CONSONANT SIGN MEDIAL RA */
819 0x1056, /* MYANMAR VOWEL SIGN VOCALIC R */
820 0x1057, /* MYANMAR VOWEL SIGN VOCALIC RR */
821 0x1062, /* MYANMAR VOWEL SIGN SGAW KAREN EU */
822 0x1063, /* MYANMAR TONE MARK SGAW KAREN HATHI */
823 0x1064, /* MYANMAR TONE MARK SGAW KAREN KE PHO */
824 0x1067, /* MYANMAR VOWEL SIGN WESTERN PWO KAREN EU */
825 0x1068, /* MYANMAR VOWEL SIGN WESTERN PWO KAREN UE */
826 0x1069, /* MYANMAR SIGN WESTERN PWO KAREN TONE-1 */
827 0x106A, /* MYANMAR SIGN WESTERN PWO KAREN TONE-2 */
828 0x106B, /* MYANMAR SIGN WESTERN PWO KAREN TONE-3 */
829 0x106C, /* MYANMAR SIGN WESTERN PWO KAREN TONE-4 */
830 0x106D, /* MYANMAR SIGN WESTERN PWO KAREN TONE-5 */
831 0x1083, /* MYANMAR VOWEL SIGN SHAN AA */
832 0x1084, /* MYANMAR VOWEL SIGN SHAN E */
833 0x1087, /* MYANMAR SIGN SHAN TONE-2 */
834 0x1088, /* MYANMAR SIGN SHAN TONE-3 */
835 0x1089, /* MYANMAR SIGN SHAN TONE-5 */
836 0x108A, /* MYANMAR SIGN SHAN TONE-6 */
837 0x108B, /* MYANMAR SIGN SHAN COUNCIL TONE-2 */
838 0x108C, /* MYANMAR SIGN SHAN COUNCIL TONE-3 */
839 0x108F, /* MYANMAR SIGN RUMAI PALAUNG TONE-5 */
840 0x17B6, /* KHMER VOWEL SIGN AA */
841 0x17BE, /* KHMER VOWEL SIGN OE */
842 0x17BF, /* KHMER VOWEL SIGN YA */
843 0x17C0, /* KHMER VOWEL SIGN IE */
844 0x17C1, /* KHMER VOWEL SIGN E */
845 0x17C2, /* KHMER VOWEL SIGN AE */
846 0x17C3, /* KHMER VOWEL SIGN AI */
847 0x17C4, /* KHMER VOWEL SIGN OO */
848 0x17C5, /* KHMER VOWEL SIGN AU */
849 0x17C7, /* KHMER SIGN REAHMUK */
850 0x17C8, /* KHMER SIGN YUUKALEAPINTU */
851 0x1923, /* LIMBU VOWEL SIGN EE */
852 0x1924, /* LIMBU VOWEL SIGN AI */
853 0x1925, /* LIMBU VOWEL SIGN OO */
854 0x1926, /* LIMBU VOWEL SIGN AU */
855 0x1929, /* LIMBU SUBJOINED LETTER YA */
856 0x192A, /* LIMBU SUBJOINED LETTER RA */
857 0x192B, /* LIMBU SUBJOINED LETTER WA */
858 0x1930, /* LIMBU SMALL LETTER KA */
859 0x1931, /* LIMBU SMALL LETTER NGA */
860 0x1933, /* LIMBU SMALL LETTER TA */
861 0x1934, /* LIMBU SMALL LETTER NA */
862 0x1935, /* LIMBU SMALL LETTER PA */
863 0x1936, /* LIMBU SMALL LETTER MA */
864 0x1937, /* LIMBU SMALL LETTER RA */
865 0x1938, /* LIMBU SMALL LETTER LA */
866 0x19B0, /* NEW TAI LUE VOWEL SIGN VOWEL SHORTENER */
867 0x19B1, /* NEW TAI LUE VOWEL SIGN AA */
868 0x19B2, /* NEW TAI LUE VOWEL SIGN II */
869 0x19B3, /* NEW TAI LUE VOWEL SIGN U */
870 0x19B4, /* NEW TAI LUE VOWEL SIGN UU */
871 0x19B5, /* NEW TAI LUE VOWEL SIGN E */
872 0x19B6, /* NEW TAI LUE VOWEL SIGN AE */
873 0x19B7, /* NEW TAI LUE VOWEL SIGN O */
874 0x19B8, /* NEW TAI LUE VOWEL SIGN OA */
875 0x19B9, /* NEW TAI LUE VOWEL SIGN UE */
876 0x19BA, /* NEW TAI LUE VOWEL SIGN AY */
877 0x19BB, /* NEW TAI LUE VOWEL SIGN AAY */
878 0x19BC, /* NEW TAI LUE VOWEL SIGN UY */
879 0x19BD, /* NEW TAI LUE VOWEL SIGN OY */
880 0x19BE, /* NEW TAI LUE VOWEL SIGN OAY */
881 0x19BF, /* NEW TAI LUE VOWEL SIGN UEY */
882 0x19C0, /* NEW TAI LUE VOWEL SIGN IY */
883 0x19C8, /* NEW TAI LUE TONE MARK-1 */
884 0x19C9, /* NEW TAI LUE TONE MARK-2 */
885 0x1A19, /* BUGINESE VOWEL SIGN E */
886 0x1A1A, /* BUGINESE VOWEL SIGN O */
887 0x1A1B, /* BUGINESE VOWEL SIGN AE */
888 0x1B04, /* BALINESE SIGN BISAH */
889 0x1B35, /* BALINESE VOWEL SIGN TEDUNG */
890 0x1B3B, /* BALINESE VOWEL SIGN RA REPA TEDUNG */
891 0x1B3D, /* BALINESE VOWEL SIGN LA LENGA TEDUNG */
892 0x1B3E, /* BALINESE VOWEL SIGN TALING */
893 0x1B3F, /* BALINESE VOWEL SIGN TALING REPA */
894 0x1B40, /* BALINESE VOWEL SIGN TALING TEDUNG */
895 0x1B41, /* BALINESE VOWEL SIGN TALING REPA TEDUNG */
896 0x1B43, /* BALINESE VOWEL SIGN PEPET TEDUNG */
897 0x1B44, /* BALINESE ADEG ADEG */
898 0x1B82, /* SUNDANESE SIGN PANGWISAD */
899 0x1BA1, /* SUNDANESE CONSONANT SIGN PAMINGKAL */
900 0x1BA6, /* SUNDANESE VOWEL SIGN PANAELAENG */
901 0x1BA7, /* SUNDANESE VOWEL SIGN PANOLONG */
902 0x1BAA, /* SUNDANESE SIGN PAMAAEH */
903 0x1C24, /* LEPCHA SUBJOINED LETTER YA */
904 0x1C25, /* LEPCHA SUBJOINED LETTER RA */
905 0x1C26, /* LEPCHA VOWEL SIGN AA */
906 0x1C27, /* LEPCHA VOWEL SIGN I */
907 0x1C28, /* LEPCHA VOWEL SIGN O */
908 0x1C29, /* LEPCHA VOWEL SIGN OO */
909 0x1C2A, /* LEPCHA VOWEL SIGN U */
910 0x1C2B, /* LEPCHA VOWEL SIGN UU */
911 0x1C34, /* LEPCHA CONSONANT SIGN NYIN-DO */
912 0x1C35, /* LEPCHA CONSONANT SIGN KANG */
913 0xA823, /* SYLOTI NAGRI VOWEL SIGN A */
914 0xA824, /* SYLOTI NAGRI VOWEL SIGN I */
915 0xA827, /* SYLOTI NAGRI VOWEL SIGN OO */
916 0xA880, /* SAURASHTRA SIGN ANUSVARA */
917 0xA881, /* SAURASHTRA SIGN VISARGA */
918 0xA8B4, /* SAURASHTRA CONSONANT SIGN HAARU */
919 0xA8B5, /* SAURASHTRA VOWEL SIGN AA */
920 0xA8B6, /* SAURASHTRA VOWEL SIGN I */
921 0xA8B7, /* SAURASHTRA VOWEL SIGN II */
922 0xA8B8, /* SAURASHTRA VOWEL SIGN U */
923 0xA8B9, /* SAURASHTRA VOWEL SIGN UU */
924 0xA8BA, /* SAURASHTRA VOWEL SIGN VOCALIC R */
925 0xA8BB, /* SAURASHTRA VOWEL SIGN VOCALIC RR */
926 0xA8BC, /* SAURASHTRA VOWEL SIGN VOCALIC L */
927 0xA8BD, /* SAURASHTRA VOWEL SIGN VOCALIC LL */
928 0xA8BE, /* SAURASHTRA VOWEL SIGN E */
929 0xA8BF, /* SAURASHTRA VOWEL SIGN EE */
930 0xA8C0, /* SAURASHTRA VOWEL SIGN AI */
931 0xA8C1, /* SAURASHTRA VOWEL SIGN O */
932 0xA8C2, /* SAURASHTRA VOWEL SIGN OO */
933 0xA8C3, /* SAURASHTRA VOWEL SIGN AU */
934 0xA952, /* REJANG CONSONANT SIGN H */
935 0xA953, /* REJANG VIRAMA */
936 0xAA2F, /* CHAM VOWEL SIGN O */
937 0xAA30, /* CHAM VOWEL SIGN AI */
938 0xAA33, /* CHAM CONSONANT SIGN YA */
939 0xAA34, /* CHAM CONSONANT SIGN RA */
940 0xAA4D /* CHAM CONSONANT SIGN FINAL H */
941 };
942 const pg_wchar *StopLow = strange_letter,
943 *StopHigh = strange_letter + lengthof(strange_letter),
944 *StopMiddle;
945 pg_wchar c;
946
947 if (prs->pgwstr)
948 c = *(prs->pgwstr + prs->state->poschar);
949 else
950 c = (pg_wchar) *(prs->wstr + prs->state->poschar);
951
952 while (StopLow < StopHigh)
953 {
954 StopMiddle = StopLow + ((StopHigh - StopLow) >> 1);
955 if (*StopMiddle == c)
956 return 1;
957 else if (*StopMiddle < c)
958 StopLow = StopMiddle + 1;
959 else
960 StopHigh = StopMiddle;
961 }
962 }
963
964 return 0;
965 }
966
967 /*
968 * Table of state/action of parser
969 */
970
971 static const TParserStateActionItem actionTPS_Base[] = {
972 {p_isEOF, 0, A_NEXT, TPS_Null, 0, NULL},
973 {p_iseqC, '<', A_PUSH, TPS_InTagFirst, 0, NULL},
974 {p_isignore, 0, A_NEXT, TPS_InSpace, 0, NULL},
975 {p_isasclet, 0, A_NEXT, TPS_InAsciiWord, 0, NULL},
976 {p_isalpha, 0, A_NEXT, TPS_InWord, 0, NULL},
977 {p_isdigit, 0, A_NEXT, TPS_InUnsignedInt, 0, NULL},
978 {p_iseqC, '-', A_PUSH, TPS_InSignedIntFirst, 0, NULL},
979 {p_iseqC, '+', A_PUSH, TPS_InSignedIntFirst, 0, NULL},
980 {p_iseqC, '&', A_PUSH, TPS_InXMLEntityFirst, 0, NULL},
981 {p_iseqC, '~', A_PUSH, TPS_InFileTwiddle, 0, NULL},
982 {p_iseqC, '/', A_PUSH, TPS_InFileFirst, 0, NULL},
983 {p_iseqC, '.', A_PUSH, TPS_InPathFirstFirst, 0, NULL},
984 {NULL, 0, A_NEXT, TPS_InSpace, 0, NULL}
985 };
986
987
988 static const TParserStateActionItem actionTPS_InNumWord[] = {
989 {p_isEOF, 0, A_BINGO, TPS_Base, NUMWORD, NULL},
990 {p_isalnum, 0, A_NEXT, TPS_InNumWord, 0, NULL},
991 {p_isspecial, 0, A_NEXT, TPS_InNumWord, 0, NULL},
992 {p_iseqC, '@', A_PUSH, TPS_InEmail, 0, NULL},
993 {p_iseqC, '/', A_PUSH, TPS_InFileFirst, 0, NULL},
994 {p_iseqC, '.', A_PUSH, TPS_InFileNext, 0, NULL},
995 {p_iseqC, '-', A_PUSH, TPS_InHyphenNumWordFirst, 0, NULL},
996 {NULL, 0, A_BINGO, TPS_Base, NUMWORD, NULL}
997 };
998
999 static const TParserStateActionItem actionTPS_InAsciiWord[] = {
1000 {p_isEOF, 0, A_BINGO, TPS_Base, ASCIIWORD, NULL},
1001 {p_isasclet, 0, A_NEXT, TPS_Null, 0, NULL},
1002 {p_iseqC, '.', A_PUSH, TPS_InHostFirstDomain, 0, NULL},
1003 {p_iseqC, '.', A_PUSH, TPS_InFileNext, 0, NULL},
1004 {p_iseqC, '-', A_PUSH, TPS_InHostFirstAN, 0, NULL},
1005 {p_iseqC, '-', A_PUSH, TPS_InHyphenAsciiWordFirst, 0, NULL},
1006 {p_iseqC, '_', A_PUSH, TPS_InHostFirstAN, 0, NULL},
1007 {p_iseqC, '@', A_PUSH, TPS_InEmail, 0, NULL},
1008 {p_iseqC, ':', A_PUSH, TPS_InProtocolFirst, 0, NULL},
1009 {p_iseqC, '/', A_PUSH, TPS_InFileFirst, 0, NULL},
1010 {p_isdigit, 0, A_PUSH, TPS_InHost, 0, NULL},
1011 {p_isdigit, 0, A_NEXT, TPS_InNumWord, 0, NULL},
1012 {p_isalpha, 0, A_NEXT, TPS_InWord, 0, NULL},
1013 {p_isspecial, 0, A_NEXT, TPS_InWord, 0, NULL},
1014 {NULL, 0, A_BINGO, TPS_Base, ASCIIWORD, NULL}
1015 };
1016
1017 static const TParserStateActionItem actionTPS_InWord[] = {
1018 {p_isEOF, 0, A_BINGO, TPS_Base, WORD_T, NULL},
1019 {p_isalpha, 0, A_NEXT, TPS_Null, 0, NULL},
1020 {p_isspecial, 0, A_NEXT, TPS_Null, 0, NULL},
1021 {p_isdigit, 0, A_NEXT, TPS_InNumWord, 0, NULL},
1022 {p_iseqC, '-', A_PUSH, TPS_InHyphenWordFirst, 0, NULL},
1023 {NULL, 0, A_BINGO, TPS_Base, WORD_T, NULL}
1024 };
1025
1026 static const TParserStateActionItem actionTPS_InUnsignedInt[] = {
1027 {p_isEOF, 0, A_BINGO, TPS_Base, UNSIGNEDINT, NULL},
1028 {p_isdigit, 0, A_NEXT, TPS_Null, 0, NULL},
1029 {p_iseqC, '.', A_PUSH, TPS_InHostFirstDomain, 0, NULL},
1030 {p_iseqC, '.', A_PUSH, TPS_InUDecimalFirst, 0, NULL},
1031 {p_iseqC, 'e', A_PUSH, TPS_InMantissaFirst, 0, NULL},
1032 {p_iseqC, 'E', A_PUSH, TPS_InMantissaFirst, 0, NULL},
1033 {p_iseqC, '-', A_PUSH, TPS_InHostFirstAN, 0, NULL},
1034 {p_iseqC, '_', A_PUSH, TPS_InHostFirstAN, 0, NULL},
1035 {p_iseqC, '@', A_PUSH, TPS_InEmail, 0, NULL},
1036 {p_isasclet, 0, A_PUSH, TPS_InHost, 0, NULL},
1037 {p_isalpha, 0, A_NEXT, TPS_InNumWord, 0, NULL},
1038 {p_isspecial, 0, A_NEXT, TPS_InNumWord, 0, NULL},
1039 {p_iseqC, '/', A_PUSH, TPS_InFileFirst, 0, NULL},
1040 {NULL, 0, A_BINGO, TPS_Base, UNSIGNEDINT, NULL}
1041 };
1042
1043 static const TParserStateActionItem actionTPS_InSignedIntFirst[] = {
1044 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1045 {p_isdigit, 0, A_NEXT | A_CLEAR, TPS_InSignedInt, 0, NULL},
1046 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1047 };
1048
1049 static const TParserStateActionItem actionTPS_InSignedInt[] = {
1050 {p_isEOF, 0, A_BINGO, TPS_Base, SIGNEDINT, NULL},
1051 {p_isdigit, 0, A_NEXT, TPS_Null, 0, NULL},
1052 {p_iseqC, '.', A_PUSH, TPS_InDecimalFirst, 0, NULL},
1053 {p_iseqC, 'e', A_PUSH, TPS_InMantissaFirst, 0, NULL},
1054 {p_iseqC, 'E', A_PUSH, TPS_InMantissaFirst, 0, NULL},
1055 {NULL, 0, A_BINGO, TPS_Base, SIGNEDINT, NULL}
1056 };
1057
1058 static const TParserStateActionItem actionTPS_InSpace[] = {
1059 {p_isEOF, 0, A_BINGO, TPS_Base, SPACE, NULL},
1060 {p_iseqC, '<', A_BINGO, TPS_Base, SPACE, NULL},
1061 {p_isignore, 0, A_NEXT, TPS_Null, 0, NULL},
1062 {p_iseqC, '-', A_BINGO, TPS_Base, SPACE, NULL},
1063 {p_iseqC, '+', A_BINGO, TPS_Base, SPACE, NULL},
1064 {p_iseqC, '&', A_BINGO, TPS_Base, SPACE, NULL},
1065 {p_iseqC, '/', A_BINGO, TPS_Base, SPACE, NULL},
1066 {p_isnotalnum, 0, A_NEXT, TPS_InSpace, 0, NULL},
1067 {NULL, 0, A_BINGO, TPS_Base, SPACE, NULL}
1068 };
1069
1070 static const TParserStateActionItem actionTPS_InUDecimalFirst[] = {
1071 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1072 {p_isdigit, 0, A_CLEAR, TPS_InUDecimal, 0, NULL},
1073 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1074 };
1075
1076 static const TParserStateActionItem actionTPS_InUDecimal[] = {
1077 {p_isEOF, 0, A_BINGO, TPS_Base, DECIMAL_T, NULL},
1078 {p_isdigit, 0, A_NEXT, TPS_InUDecimal, 0, NULL},
1079 {p_iseqC, '.', A_PUSH, TPS_InVersionFirst, 0, NULL},
1080 {p_iseqC, 'e', A_PUSH, TPS_InMantissaFirst, 0, NULL},
1081 {p_iseqC, 'E', A_PUSH, TPS_InMantissaFirst, 0, NULL},
1082 {NULL, 0, A_BINGO, TPS_Base, DECIMAL_T, NULL}
1083 };
1084
1085 static const TParserStateActionItem actionTPS_InDecimalFirst[] = {
1086 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1087 {p_isdigit, 0, A_CLEAR, TPS_InDecimal, 0, NULL},
1088 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1089 };
1090
1091 static const TParserStateActionItem actionTPS_InDecimal[] = {
1092 {p_isEOF, 0, A_BINGO, TPS_Base, DECIMAL_T, NULL},
1093 {p_isdigit, 0, A_NEXT, TPS_InDecimal, 0, NULL},
1094 {p_iseqC, '.', A_PUSH, TPS_InVerVersion, 0, NULL},
1095 {p_iseqC, 'e', A_PUSH, TPS_InMantissaFirst, 0, NULL},
1096 {p_iseqC, 'E', A_PUSH, TPS_InMantissaFirst, 0, NULL},
1097 {NULL, 0, A_BINGO, TPS_Base, DECIMAL_T, NULL}
1098 };
1099
1100 static const TParserStateActionItem actionTPS_InVerVersion[] = {
1101 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1102 {p_isdigit, 0, A_RERUN, TPS_InSVerVersion, 0, SpecialVerVersion},
1103 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1104 };
1105
1106 static const TParserStateActionItem actionTPS_InSVerVersion[] = {
1107 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1108 {p_isdigit, 0, A_BINGO | A_CLRALL, TPS_InUnsignedInt, SPACE, NULL},
1109 {NULL, 0, A_NEXT, TPS_Null, 0, NULL}
1110 };
1111
1112
1113 static const TParserStateActionItem actionTPS_InVersionFirst[] = {
1114 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1115 {p_isdigit, 0, A_CLEAR, TPS_InVersion, 0, NULL},
1116 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1117 };
1118
1119 static const TParserStateActionItem actionTPS_InVersion[] = {
1120 {p_isEOF, 0, A_BINGO, TPS_Base, VERSIONNUMBER, NULL},
1121 {p_isdigit, 0, A_NEXT, TPS_InVersion, 0, NULL},
1122 {p_iseqC, '.', A_PUSH, TPS_InVersionFirst, 0, NULL},
1123 {NULL, 0, A_BINGO, TPS_Base, VERSIONNUMBER, NULL}
1124 };
1125
1126 static const TParserStateActionItem actionTPS_InMantissaFirst[] = {
1127 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1128 {p_isdigit, 0, A_CLEAR, TPS_InMantissa, 0, NULL},
1129 {p_iseqC, '+', A_NEXT, TPS_InMantissaSign, 0, NULL},
1130 {p_iseqC, '-', A_NEXT, TPS_InMantissaSign, 0, NULL},
1131 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1132 };
1133
1134 static const TParserStateActionItem actionTPS_InMantissaSign[] = {
1135 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1136 {p_isdigit, 0, A_CLEAR, TPS_InMantissa, 0, NULL},
1137 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1138 };
1139
1140 static const TParserStateActionItem actionTPS_InMantissa[] = {
1141 {p_isEOF, 0, A_BINGO, TPS_Base, SCIENTIFIC, NULL},
1142 {p_isdigit, 0, A_NEXT, TPS_InMantissa, 0, NULL},
1143 {NULL, 0, A_BINGO, TPS_Base, SCIENTIFIC, NULL}
1144 };
1145
1146 static const TParserStateActionItem actionTPS_InXMLEntityFirst[] = {
1147 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1148 {p_iseqC, '#', A_NEXT, TPS_InXMLEntityNumFirst, 0, NULL},
1149 {p_isasclet, 0, A_NEXT, TPS_InXMLEntity, 0, NULL},
1150 {p_iseqC, ':', A_NEXT, TPS_InXMLEntity, 0, NULL},
1151 {p_iseqC, '_', A_NEXT, TPS_InXMLEntity, 0, NULL},
1152 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1153 };
1154
1155 static const TParserStateActionItem actionTPS_InXMLEntity[] = {
1156 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1157 {p_isalnum, 0, A_NEXT, TPS_InXMLEntity, 0, NULL},
1158 {p_iseqC, ':', A_NEXT, TPS_InXMLEntity, 0, NULL},
1159 {p_iseqC, '_', A_NEXT, TPS_InXMLEntity, 0, NULL},
1160 {p_iseqC, '.', A_NEXT, TPS_InXMLEntity, 0, NULL},
1161 {p_iseqC, '-', A_NEXT, TPS_InXMLEntity, 0, NULL},
1162 {p_iseqC, ';', A_NEXT, TPS_InXMLEntityEnd, 0, NULL},
1163 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1164 };
1165
1166 static const TParserStateActionItem actionTPS_InXMLEntityNumFirst[] = {
1167 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1168 {p_iseqC, 'x', A_NEXT, TPS_InXMLEntityHexNumFirst, 0, NULL},
1169 {p_iseqC, 'X', A_NEXT, TPS_InXMLEntityHexNumFirst, 0, NULL},
1170 {p_isdigit, 0, A_NEXT, TPS_InXMLEntityNum, 0, NULL},
1171 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1172 };
1173
1174 static const TParserStateActionItem actionTPS_InXMLEntityHexNumFirst[] = {
1175 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1176 {p_isxdigit, 0, A_NEXT, TPS_InXMLEntityHexNum, 0, NULL},
1177 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1178 };
1179
1180 static const TParserStateActionItem actionTPS_InXMLEntityNum[] = {
1181 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1182 {p_isdigit, 0, A_NEXT, TPS_InXMLEntityNum, 0, NULL},
1183 {p_iseqC, ';', A_NEXT, TPS_InXMLEntityEnd, 0, NULL},
1184 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1185 };
1186
1187 static const TParserStateActionItem actionTPS_InXMLEntityHexNum[] = {
1188 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1189 {p_isxdigit, 0, A_NEXT, TPS_InXMLEntityHexNum, 0, NULL},
1190 {p_iseqC, ';', A_NEXT, TPS_InXMLEntityEnd, 0, NULL},
1191 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1192 };
1193
1194 static const TParserStateActionItem actionTPS_InXMLEntityEnd[] = {
1195 {NULL, 0, A_BINGO | A_CLEAR, TPS_Base, XMLENTITY, NULL}
1196 };
1197
1198 static const TParserStateActionItem actionTPS_InTagFirst[] = {
1199 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1200 {p_iseqC, '/', A_PUSH, TPS_InTagCloseFirst, 0, NULL},
1201 {p_iseqC, '!', A_PUSH, TPS_InCommentFirst, 0, NULL},
1202 {p_iseqC, '?', A_PUSH, TPS_InXMLBegin, 0, NULL},
1203 {p_isasclet, 0, A_PUSH, TPS_InTagName, 0, NULL},
1204 {p_iseqC, ':', A_PUSH, TPS_InTagName, 0, NULL},
1205 {p_iseqC, '_', A_PUSH, TPS_InTagName, 0, NULL},
1206 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1207 };
1208
1209 static const TParserStateActionItem actionTPS_InXMLBegin[] = {
1210 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1211 /* <?xml ... */
1212 /* XXX do we wants states for the m and l ? Right now this accepts <?xZ */
1213 {p_iseqC, 'x', A_NEXT, TPS_InTag, 0, NULL},
1214 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1215 };
1216
1217 static const TParserStateActionItem actionTPS_InTagCloseFirst[] = {
1218 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1219 {p_isasclet, 0, A_NEXT, TPS_InTagName, 0, NULL},
1220 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1221 };
1222
1223 static const TParserStateActionItem actionTPS_InTagName[] = {
1224 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1225 /* <br/> case */
1226 {p_iseqC, '/', A_NEXT, TPS_InTagBeginEnd, 0, NULL},
1227 {p_iseqC, '>', A_NEXT, TPS_InTagEnd, 0, SpecialTags},
1228 {p_isspace, 0, A_NEXT, TPS_InTag, 0, SpecialTags},
1229 {p_isalnum, 0, A_NEXT, TPS_Null, 0, NULL},
1230 {p_iseqC, ':', A_NEXT, TPS_Null, 0, NULL},
1231 {p_iseqC, '_', A_NEXT, TPS_Null, 0, NULL},
1232 {p_iseqC, '.', A_NEXT, TPS_Null, 0, NULL},
1233 {p_iseqC, '-', A_NEXT, TPS_Null, 0, NULL},
1234 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1235 };
1236
1237 static const TParserStateActionItem actionTPS_InTagBeginEnd[] = {
1238 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1239 {p_iseqC, '>', A_NEXT, TPS_InTagEnd, 0, NULL},
1240 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1241 };
1242
1243 static const TParserStateActionItem actionTPS_InTag[] = {
1244 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1245 {p_iseqC, '>', A_NEXT, TPS_InTagEnd, 0, SpecialTags},
1246 {p_iseqC, '\'', A_NEXT, TPS_InTagEscapeK, 0, NULL},
1247 {p_iseqC, '"', A_NEXT, TPS_InTagEscapeKK, 0, NULL},
1248 {p_isasclet, 0, A_NEXT, TPS_Null, 0, NULL},
1249 {p_isdigit, 0, A_NEXT, TPS_Null, 0, NULL},
1250 {p_iseqC, '=', A_NEXT, TPS_Null, 0, NULL},
1251 {p_iseqC, '-', A_NEXT, TPS_Null, 0, NULL},
1252 {p_iseqC, '_', A_NEXT, TPS_Null, 0, NULL},
1253 {p_iseqC, '#', A_NEXT, TPS_Null, 0, NULL},
1254 {p_iseqC, '/', A_NEXT, TPS_Null, 0, NULL},
1255 {p_iseqC, ':', A_NEXT, TPS_Null, 0, NULL},
1256 {p_iseqC, '.', A_NEXT, TPS_Null, 0, NULL},
1257 {p_iseqC, '&', A_NEXT, TPS_Null, 0, NULL},
1258 {p_iseqC, '?', A_NEXT, TPS_Null, 0, NULL},
1259 {p_iseqC, '%', A_NEXT, TPS_Null, 0, NULL},
1260 {p_iseqC, '~', A_NEXT, TPS_Null, 0, NULL},
1261 {p_isspace, 0, A_NEXT, TPS_Null, 0, SpecialTags},
1262 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1263 };
1264
1265 static const TParserStateActionItem actionTPS_InTagEscapeK[] = {
1266 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1267 {p_iseqC, '\\', A_PUSH, TPS_InTagBackSleshed, 0, NULL},
1268 {p_iseqC, '\'', A_NEXT, TPS_InTag, 0, NULL},
1269 {NULL, 0, A_NEXT, TPS_InTagEscapeK, 0, NULL}
1270 };
1271
1272 static const TParserStateActionItem actionTPS_InTagEscapeKK[] = {
1273 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1274 {p_iseqC, '\\', A_PUSH, TPS_InTagBackSleshed, 0, NULL},
1275 {p_iseqC, '"', A_NEXT, TPS_InTag, 0, NULL},
1276 {NULL, 0, A_NEXT, TPS_InTagEscapeKK, 0, NULL}
1277 };
1278
1279 static const TParserStateActionItem actionTPS_InTagBackSleshed[] = {
1280 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1281 {NULL, 0, A_MERGE, TPS_Null, 0, NULL}
1282 };
1283
1284 static const TParserStateActionItem actionTPS_InTagEnd[] = {
1285 {NULL, 0, A_BINGO | A_CLRALL, TPS_Base, TAG_T, NULL}
1286 };
1287
1288 static const TParserStateActionItem actionTPS_InCommentFirst[] = {
1289 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1290 {p_iseqC, '-', A_NEXT, TPS_InCommentLast, 0, NULL},
1291 /* <!DOCTYPE ...> */
1292 {p_iseqC, 'D', A_NEXT, TPS_InTag, 0, NULL},
1293 {p_iseqC, 'd', A_NEXT, TPS_InTag, 0, NULL},
1294 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1295 };
1296
1297 static const TParserStateActionItem actionTPS_InCommentLast[] = {
1298 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1299 {p_iseqC, '-', A_NEXT, TPS_InComment, 0, NULL},
1300 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1301 };
1302
1303 static const TParserStateActionItem actionTPS_InComment[] = {
1304 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1305 {p_iseqC, '-', A_NEXT, TPS_InCloseCommentFirst, 0, NULL},
1306 {NULL, 0, A_NEXT, TPS_Null, 0, NULL}
1307 };
1308
1309 static const TParserStateActionItem actionTPS_InCloseCommentFirst[] = {
1310 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1311 {p_iseqC, '-', A_NEXT, TPS_InCloseCommentLast, 0, NULL},
1312 {NULL, 0, A_NEXT, TPS_InComment, 0, NULL}
1313 };
1314
1315 static const TParserStateActionItem actionTPS_InCloseCommentLast[] = {
1316 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1317 {p_iseqC, '-', A_NEXT, TPS_Null, 0, NULL},
1318 {p_iseqC, '>', A_NEXT, TPS_InCommentEnd, 0, NULL},
1319 {NULL, 0, A_NEXT, TPS_InComment, 0, NULL}
1320 };
1321
1322 static const TParserStateActionItem actionTPS_InCommentEnd[] = {
1323 {NULL, 0, A_BINGO | A_CLRALL, TPS_Base, TAG_T, NULL}
1324 };
1325
1326 static const TParserStateActionItem actionTPS_InHostFirstDomain[] = {
1327 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1328 {p_isasclet, 0, A_NEXT, TPS_InHostDomainSecond, 0, NULL},
1329 {p_isdigit, 0, A_NEXT, TPS_InHost, 0, NULL},
1330 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1331 };
1332
1333 static const TParserStateActionItem actionTPS_InHostDomainSecond[] = {
1334 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1335 {p_isasclet, 0, A_NEXT, TPS_InHostDomain, 0, NULL},
1336 {p_isdigit, 0, A_PUSH, TPS_InHost, 0, NULL},
1337 {p_iseqC, '-', A_PUSH, TPS_InHostFirstAN, 0, NULL},
1338 {p_iseqC, '_', A_PUSH, TPS_InHostFirstAN, 0, NULL},
1339 {p_iseqC, '.', A_PUSH, TPS_InHostFirstDomain, 0, NULL},
1340 {p_iseqC, '@', A_PUSH, TPS_InEmail, 0, NULL},
1341 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1342 };
1343
1344 static const TParserStateActionItem actionTPS_InHostDomain[] = {
1345 {p_isEOF, 0, A_BINGO | A_CLRALL, TPS_Base, HOST, NULL},
1346 {p_isasclet, 0, A_NEXT, TPS_InHostDomain, 0, NULL},
1347 {p_isdigit, 0, A_PUSH, TPS_InHost, 0, NULL},
1348 {p_iseqC, ':', A_PUSH, TPS_InPortFirst, 0, NULL},
1349 {p_iseqC, '-', A_PUSH, TPS_InHostFirstAN, 0, NULL},
1350 {p_iseqC, '_', A_PUSH, TPS_InHostFirstAN, 0, NULL},
1351 {p_iseqC, '.', A_PUSH, TPS_InHostFirstDomain, 0, NULL},
1352 {p_iseqC, '@', A_PUSH, TPS_InEmail, 0, NULL},
1353 {p_isdigit, 0, A_POP, TPS_Null, 0, NULL},
1354 {p_isstophost, 0, A_BINGO | A_CLRALL, TPS_InURLPathStart, HOST, NULL},
1355 {p_iseqC, '/', A_PUSH, TPS_InFURL, 0, NULL},
1356 {NULL, 0, A_BINGO | A_CLRALL, TPS_Base, HOST, NULL}
1357 };
1358
1359 static const TParserStateActionItem actionTPS_InPortFirst[] = {
1360 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1361 {p_isdigit, 0, A_NEXT, TPS_InPort, 0, NULL},
1362 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1363 };
1364
1365 static const TParserStateActionItem actionTPS_InPort[] = {
1366 {p_isEOF, 0, A_BINGO | A_CLRALL, TPS_Base, HOST, NULL},
1367 {p_isdigit, 0, A_NEXT, TPS_InPort, 0, NULL},
1368 {p_isstophost, 0, A_BINGO | A_CLRALL, TPS_InURLPathStart, HOST, NULL},
1369 {p_iseqC, '/', A_PUSH, TPS_InFURL, 0, NULL},
1370 {NULL, 0, A_BINGO | A_CLRALL, TPS_Base, HOST, NULL}
1371 };
1372
1373 static const TParserStateActionItem actionTPS_InHostFirstAN[] = {
1374 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1375 {p_isdigit, 0, A_NEXT, TPS_InHost, 0, NULL},
1376 {p_isasclet, 0, A_NEXT, TPS_InHost, 0, NULL},
1377 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1378 };
1379
1380 static const TParserStateActionItem actionTPS_InHost[] = {
1381 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1382 {p_isdigit, 0, A_NEXT, TPS_InHost, 0, NULL},
1383 {p_isasclet, 0, A_NEXT, TPS_InHost, 0, NULL},
1384 {p_iseqC, '@', A_PUSH, TPS_InEmail, 0, NULL},
1385 {p_iseqC, '.', A_PUSH, TPS_InHostFirstDomain, 0, NULL},
1386 {p_iseqC, '-', A_PUSH, TPS_InHostFirstAN, 0, NULL},
1387 {p_iseqC, '_', A_PUSH, TPS_InHostFirstAN, 0, NULL},
1388 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1389 };
1390
1391 static const TParserStateActionItem actionTPS_InEmail[] = {
1392 {p_isstophost, 0, A_POP, TPS_Null, 0, NULL},
1393 {p_ishost, 0, A_BINGO | A_CLRALL, TPS_Base, EMAIL, NULL},
1394 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1395 };
1396
1397 static const TParserStateActionItem actionTPS_InFileFirst[] = {
1398 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1399 {p_isasclet, 0, A_NEXT, TPS_InFile, 0, NULL},
1400 {p_isdigit, 0, A_NEXT, TPS_InFile, 0, NULL},
1401 {p_iseqC, '.', A_NEXT, TPS_InPathFirst, 0, NULL},
1402 {p_iseqC, '_', A_NEXT, TPS_InFile, 0, NULL},
1403 {p_iseqC, '~', A_PUSH, TPS_InFileTwiddle, 0, NULL},
1404 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1405 };
1406
1407 static const TParserStateActionItem actionTPS_InFileTwiddle[] = {
1408 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1409 {p_isasclet, 0, A_NEXT, TPS_InFile, 0, NULL},
1410 {p_isdigit, 0, A_NEXT, TPS_InFile, 0, NULL},
1411 {p_iseqC, '_', A_NEXT, TPS_InFile, 0, NULL},
1412 {p_iseqC, '/', A_NEXT, TPS_InFileFirst, 0, NULL},
1413 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1414 };
1415
1416 static const TParserStateActionItem actionTPS_InPathFirst[] = {
1417 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1418 {p_isasclet, 0, A_NEXT, TPS_InFile, 0, NULL},
1419 {p_isdigit, 0, A_NEXT, TPS_InFile, 0, NULL},
1420 {p_iseqC, '_', A_NEXT, TPS_InFile, 0, NULL},
1421 {p_iseqC, '.', A_NEXT, TPS_InPathSecond, 0, NULL},
1422 {p_iseqC, '/', A_NEXT, TPS_InFileFirst, 0, NULL},
1423 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1424 };
1425
1426 static const TParserStateActionItem actionTPS_InPathFirstFirst[] = {
1427 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1428 {p_iseqC, '.', A_NEXT, TPS_InPathSecond, 0, NULL},
1429 {p_iseqC, '/', A_NEXT, TPS_InFileFirst, 0, NULL},
1430 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1431 };
1432
1433 static const TParserStateActionItem actionTPS_InPathSecond[] = {
1434 {p_isEOF, 0, A_BINGO | A_CLEAR, TPS_Base, FILEPATH, NULL},
1435 {p_iseqC, '/', A_NEXT | A_PUSH, TPS_InFileFirst, 0, NULL},
1436 {p_iseqC, '/', A_BINGO | A_CLEAR, TPS_Base, FILEPATH, NULL},
1437 {p_isspace, 0, A_BINGO | A_CLEAR, TPS_Base, FILEPATH, NULL},
1438 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1439 };
1440
1441 static const TParserStateActionItem actionTPS_InFile[] = {
1442 {p_isEOF, 0, A_BINGO, TPS_Base, FILEPATH, NULL},
1443 {p_isasclet, 0, A_NEXT, TPS_InFile, 0, NULL},
1444 {p_isdigit, 0, A_NEXT, TPS_InFile, 0, NULL},
1445 {p_iseqC, '.', A_PUSH, TPS_InFileNext, 0, NULL},
1446 {p_iseqC, '_', A_NEXT, TPS_InFile, 0, NULL},
1447 {p_iseqC, '-', A_NEXT, TPS_InFile, 0, NULL},
1448 {p_iseqC, '/', A_PUSH, TPS_InFileFirst, 0, NULL},
1449 {NULL, 0, A_BINGO, TPS_Base, FILEPATH, NULL}
1450 };
1451
1452 static const TParserStateActionItem actionTPS_InFileNext[] = {
1453 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1454 {p_isasclet, 0, A_CLEAR, TPS_InFile, 0, NULL},
1455 {p_isdigit, 0, A_CLEAR, TPS_InFile, 0, NULL},
1456 {p_iseqC, '_', A_CLEAR, TPS_InFile, 0, NULL},
1457 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1458 };
1459
1460 static const TParserStateActionItem actionTPS_InURLPathFirst[] = {
1461 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1462 {p_isurlchar, 0, A_NEXT, TPS_InURLPath, 0, NULL},
1463 {NULL, 0, A_POP, TPS_Null, 0, NULL},
1464 };
1465
1466 static const TParserStateActionItem actionTPS_InURLPathStart[] = {
1467 {NULL, 0, A_NEXT, TPS_InURLPath, 0, NULL}
1468 };
1469
1470 static const TParserStateActionItem actionTPS_InURLPath[] = {
1471 {p_isEOF, 0, A_BINGO, TPS_Base, URLPATH, NULL},
1472 {p_isurlchar, 0, A_NEXT, TPS_InURLPath, 0, NULL},
1473 {NULL, 0, A_BINGO, TPS_Base, URLPATH, NULL}
1474 };
1475
1476 static const TParserStateActionItem actionTPS_InFURL[] = {
1477 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1478 {p_isURLPath, 0, A_BINGO | A_CLRALL, TPS_Base, URL_T, SpecialFURL},
1479 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1480 };
1481
1482 static const TParserStateActionItem actionTPS_InProtocolFirst[] = {
1483 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1484 {p_iseqC, '/', A_NEXT, TPS_InProtocolSecond, 0, NULL},
1485 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1486 };
1487
1488 static const TParserStateActionItem actionTPS_InProtocolSecond[] = {
1489 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1490 {p_iseqC, '/', A_NEXT, TPS_InProtocolEnd, 0, NULL},
1491 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1492 };
1493
1494 static const TParserStateActionItem actionTPS_InProtocolEnd[] = {
1495 {NULL, 0, A_BINGO | A_CLRALL, TPS_Base, PROTOCOL, NULL}
1496 };
1497
1498 static const TParserStateActionItem actionTPS_InHyphenAsciiWordFirst[] = {
1499 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1500 {p_isasclet, 0, A_NEXT, TPS_InHyphenAsciiWord, 0, NULL},
1501 {p_isalpha, 0, A_NEXT, TPS_InHyphenWord, 0, NULL},
1502 {p_isdigit, 0, A_NEXT, TPS_InHyphenDigitLookahead, 0, NULL},
1503 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1504 };
1505
1506 static const TParserStateActionItem actionTPS_InHyphenAsciiWord[] = {
1507 {p_isEOF, 0, A_BINGO | A_CLRALL, TPS_InParseHyphen, ASCIIHWORD, SpecialHyphen},
1508 {p_isasclet, 0, A_NEXT, TPS_InHyphenAsciiWord, 0, NULL},
1509 {p_isalpha, 0, A_NEXT, TPS_InHyphenWord, 0, NULL},
1510 {p_isspecial, 0, A_NEXT, TPS_InHyphenWord, 0, NULL},
1511 {p_isdigit, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL},
1512 {p_iseqC, '-', A_PUSH, TPS_InHyphenAsciiWordFirst, 0, NULL},
1513 {NULL, 0, A_BINGO | A_CLRALL, TPS_InParseHyphen, ASCIIHWORD, SpecialHyphen}
1514 };
1515
1516 static const TParserStateActionItem actionTPS_InHyphenWordFirst[] = {
1517 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1518 {p_isalpha, 0, A_NEXT, TPS_InHyphenWord, 0, NULL},
1519 {p_isdigit, 0, A_NEXT, TPS_InHyphenDigitLookahead, 0, NULL},
1520 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1521 };
1522
1523 static const TParserStateActionItem actionTPS_InHyphenWord[] = {
1524 {p_isEOF, 0, A_BINGO | A_CLRALL, TPS_InParseHyphen, HWORD, SpecialHyphen},
1525 {p_isalpha, 0, A_NEXT, TPS_InHyphenWord, 0, NULL},
1526 {p_isspecial, 0, A_NEXT, TPS_InHyphenWord, 0, NULL},
1527 {p_isdigit, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL},
1528 {p_iseqC, '-', A_PUSH, TPS_InHyphenWordFirst, 0, NULL},
1529 {NULL, 0, A_BINGO | A_CLRALL, TPS_InParseHyphen, HWORD, SpecialHyphen}
1530 };
1531
1532 static const TParserStateActionItem actionTPS_InHyphenNumWordFirst[] = {
1533 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1534 {p_isalpha, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL},
1535 {p_isdigit, 0, A_NEXT, TPS_InHyphenDigitLookahead, 0, NULL},
1536 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1537 };
1538
1539 static const TParserStateActionItem actionTPS_InHyphenNumWord[] = {
1540 {p_isEOF, 0, A_BINGO | A_CLRALL, TPS_InParseHyphen, NUMHWORD, SpecialHyphen},
1541 {p_isalnum, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL},
1542 {p_isspecial, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL},
1543 {p_iseqC, '-', A_PUSH, TPS_InHyphenNumWordFirst, 0, NULL},
1544 {NULL, 0, A_BINGO | A_CLRALL, TPS_InParseHyphen, NUMHWORD, SpecialHyphen}
1545 };
1546
1547 static const TParserStateActionItem actionTPS_InHyphenDigitLookahead[] = {
1548 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1549 {p_isdigit, 0, A_NEXT, TPS_InHyphenDigitLookahead, 0, NULL},
1550 {p_isalpha, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL},
1551 {p_isspecial, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL},
1552 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1553 };
1554
1555 static const TParserStateActionItem actionTPS_InParseHyphen[] = {
1556 {p_isEOF, 0, A_RERUN, TPS_Base, 0, NULL},
1557 {p_isasclet, 0, A_NEXT, TPS_InHyphenAsciiWordPart, 0, NULL},
1558 {p_isalpha, 0, A_NEXT, TPS_InHyphenWordPart, 0, NULL},
1559 {p_isdigit, 0, A_PUSH, TPS_InHyphenUnsignedInt, 0, NULL},
1560 {p_iseqC, '-', A_PUSH, TPS_InParseHyphenHyphen, 0, NULL},
1561 {NULL, 0, A_RERUN, TPS_Base, 0, NULL}
1562 };
1563
1564 static const TParserStateActionItem actionTPS_InParseHyphenHyphen[] = {
1565 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1566 {p_isalnum, 0, A_BINGO | A_CLEAR, TPS_InParseHyphen, SPACE, NULL},
1567 {p_isspecial, 0, A_BINGO | A_CLEAR, TPS_InParseHyphen, SPACE, NULL},
1568 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1569 };
1570
1571 static const TParserStateActionItem actionTPS_InHyphenWordPart[] = {
1572 {p_isEOF, 0, A_BINGO, TPS_Base, PARTHWORD, NULL},
1573 {p_isalpha, 0, A_NEXT, TPS_InHyphenWordPart, 0, NULL},
1574 {p_isspecial, 0, A_NEXT, TPS_InHyphenWordPart, 0, NULL},
1575 {p_isdigit, 0, A_NEXT, TPS_InHyphenNumWordPart, 0, NULL},
1576 {NULL, 0, A_BINGO, TPS_InParseHyphen, PARTHWORD, NULL}
1577 };
1578
1579 static const TParserStateActionItem actionTPS_InHyphenAsciiWordPart[] = {
1580 {p_isEOF, 0, A_BINGO, TPS_Base, ASCIIPARTHWORD, NULL},
1581 {p_isasclet, 0, A_NEXT, TPS_InHyphenAsciiWordPart, 0, NULL},
1582 {p_isalpha, 0, A_NEXT, TPS_InHyphenWordPart, 0, NULL},
1583 {p_isspecial, 0, A_NEXT, TPS_InHyphenWordPart, 0, NULL},
1584 {p_isdigit, 0, A_NEXT, TPS_InHyphenNumWordPart, 0, NULL},
1585 {NULL, 0, A_BINGO, TPS_InParseHyphen, ASCIIPARTHWORD, NULL}
1586 };
1587
1588 static const TParserStateActionItem actionTPS_InHyphenNumWordPart[] = {
1589 {p_isEOF, 0, A_BINGO, TPS_Base, NUMPARTHWORD, NULL},
1590 {p_isalnum, 0, A_NEXT, TPS_InHyphenNumWordPart, 0, NULL},
1591 {p_isspecial, 0, A_NEXT, TPS_InHyphenNumWordPart, 0, NULL},
1592 {NULL, 0, A_BINGO, TPS_InParseHyphen, NUMPARTHWORD, NULL}
1593 };
1594
1595 static const TParserStateActionItem actionTPS_InHyphenUnsignedInt[] = {
1596 {p_isEOF, 0, A_POP, TPS_Null, 0, NULL},
1597 {p_isdigit, 0, A_NEXT, TPS_Null, 0, NULL},
1598 {p_isalpha, 0, A_CLEAR, TPS_InHyphenNumWordPart, 0, NULL},
1599 {p_isspecial, 0, A_CLEAR, TPS_InHyphenNumWordPart, 0, NULL},
1600 {NULL, 0, A_POP, TPS_Null, 0, NULL}
1601 };
1602
1603
1604 /*
1605 * main table of per-state parser actions
1606 */
1607 typedef struct
1608 {
1609 const TParserStateActionItem *action; /* the actual state info */
1610 TParserState state; /* only for Assert crosscheck */
1611 #ifdef WPARSER_TRACE
1612 const char *state_name; /* only for debug printout */
1613 #endif
1614 } TParserStateAction;
1615
1616 #ifdef WPARSER_TRACE
1617 #define TPARSERSTATEACTION(state) \
1618 { CppConcat(action,state), state, CppAsString(state) }
1619 #else
1620 #define TPARSERSTATEACTION(state) \
1621 { CppConcat(action,state), state }
1622 #endif
1623
1624 /*
1625 * order must be the same as in typedef enum {} TParserState!!
1626 */
1627
1628 static const TParserStateAction Actions[] = {
1629 TPARSERSTATEACTION(TPS_Base),
1630 TPARSERSTATEACTION(TPS_InNumWord),
1631 TPARSERSTATEACTION(TPS_InAsciiWord),
1632 TPARSERSTATEACTION(TPS_InWord),
1633 TPARSERSTATEACTION(TPS_InUnsignedInt),
1634 TPARSERSTATEACTION(TPS_InSignedIntFirst),
1635 TPARSERSTATEACTION(TPS_InSignedInt),
1636 TPARSERSTATEACTION(TPS_InSpace),
1637 TPARSERSTATEACTION(TPS_InUDecimalFirst),
1638 TPARSERSTATEACTION(TPS_InUDecimal),
1639 TPARSERSTATEACTION(TPS_InDecimalFirst),
1640 TPARSERSTATEACTION(TPS_InDecimal),
1641 TPARSERSTATEACTION(TPS_InVerVersion),
1642 TPARSERSTATEACTION(TPS_InSVerVersion),
1643 TPARSERSTATEACTION(TPS_InVersionFirst),
1644 TPARSERSTATEACTION(TPS_InVersion),
1645 TPARSERSTATEACTION(TPS_InMantissaFirst),
1646 TPARSERSTATEACTION(TPS_InMantissaSign),
1647 TPARSERSTATEACTION(TPS_InMantissa),
1648 TPARSERSTATEACTION(TPS_InXMLEntityFirst),
1649 TPARSERSTATEACTION(TPS_InXMLEntity),
1650 TPARSERSTATEACTION(TPS_InXMLEntityNumFirst),
1651 TPARSERSTATEACTION(TPS_InXMLEntityNum),
1652 TPARSERSTATEACTION(TPS_InXMLEntityHexNumFirst),
1653 TPARSERSTATEACTION(TPS_InXMLEntityHexNum),
1654 TPARSERSTATEACTION(TPS_InXMLEntityEnd),
1655 TPARSERSTATEACTION(TPS_InTagFirst),
1656 TPARSERSTATEACTION(TPS_InXMLBegin),
1657 TPARSERSTATEACTION(TPS_InTagCloseFirst),
1658 TPARSERSTATEACTION(TPS_InTagName),
1659 TPARSERSTATEACTION(TPS_InTagBeginEnd),
1660 TPARSERSTATEACTION(TPS_InTag),
1661 TPARSERSTATEACTION(TPS_InTagEscapeK),
1662 TPARSERSTATEACTION(TPS_InTagEscapeKK),
1663 TPARSERSTATEACTION(TPS_InTagBackSleshed),
1664 TPARSERSTATEACTION(TPS_InTagEnd),
1665 TPARSERSTATEACTION(TPS_InCommentFirst),
1666 TPARSERSTATEACTION(TPS_InCommentLast),
1667 TPARSERSTATEACTION(TPS_InComment),
1668 TPARSERSTATEACTION(TPS_InCloseCommentFirst),
1669 TPARSERSTATEACTION(TPS_InCloseCommentLast),
1670 TPARSERSTATEACTION(TPS_InCommentEnd),
1671 TPARSERSTATEACTION(TPS_InHostFirstDomain),
1672 TPARSERSTATEACTION(TPS_InHostDomainSecond),
1673 TPARSERSTATEACTION(TPS_InHostDomain),
1674 TPARSERSTATEACTION(TPS_InPortFirst),
1675 TPARSERSTATEACTION(TPS_InPort),
1676 TPARSERSTATEACTION(TPS_InHostFirstAN),
1677 TPARSERSTATEACTION(TPS_InHost),
1678 TPARSERSTATEACTION(TPS_InEmail),
1679 TPARSERSTATEACTION(TPS_InFileFirst),
1680 TPARSERSTATEACTION(TPS_InFileTwiddle),
1681 TPARSERSTATEACTION(TPS_InPathFirst),
1682 TPARSERSTATEACTION(TPS_InPathFirstFirst),
1683 TPARSERSTATEACTION(TPS_InPathSecond),
1684 TPARSERSTATEACTION(TPS_InFile),
1685 TPARSERSTATEACTION(TPS_InFileNext),
1686 TPARSERSTATEACTION(TPS_InURLPathFirst),
1687 TPARSERSTATEACTION(TPS_InURLPathStart),
1688 TPARSERSTATEACTION(TPS_InURLPath),
1689 TPARSERSTATEACTION(TPS_InFURL),
1690 TPARSERSTATEACTION(TPS_InProtocolFirst),
1691 TPARSERSTATEACTION(TPS_InProtocolSecond),
1692 TPARSERSTATEACTION(TPS_InProtocolEnd),
1693 TPARSERSTATEACTION(TPS_InHyphenAsciiWordFirst),
1694 TPARSERSTATEACTION(TPS_InHyphenAsciiWord),
1695 TPARSERSTATEACTION(TPS_InHyphenWordFirst),
1696 TPARSERSTATEACTION(TPS_InHyphenWord),
1697 TPARSERSTATEACTION(TPS_InHyphenNumWordFirst),
1698 TPARSERSTATEACTION(TPS_InHyphenNumWord),
1699 TPARSERSTATEACTION(TPS_InHyphenDigitLookahead),
1700 TPARSERSTATEACTION(TPS_InParseHyphen),
1701 TPARSERSTATEACTION(TPS_InParseHyphenHyphen),
1702 TPARSERSTATEACTION(TPS_InHyphenWordPart),
1703 TPARSERSTATEACTION(TPS_InHyphenAsciiWordPart),
1704 TPARSERSTATEACTION(TPS_InHyphenNumWordPart),
1705 TPARSERSTATEACTION(TPS_InHyphenUnsignedInt)
1706 };
1707
1708
1709 static bool
1710 TParserGet(TParser *prs)
1711 {
1712 const TParserStateActionItem *item = NULL;
1713
1714 CHECK_FOR_INTERRUPTS();
1715
1716 Assert(prs->state);
1717
1718 if (prs->state->posbyte >= prs->lenstr)
1719 return false;
1720
1721 prs->token = prs->str + prs->state->posbyte;
1722 prs->state->pushedAtAction = NULL;
1723
1724 /* look at string */
1725 while (prs->state->posbyte <= prs->lenstr)
1726 {
1727 if (prs->state->posbyte == prs->lenstr)
1728 prs->state->charlen = 0;
1729 else
1730 prs->state->charlen = (prs->charmaxlen == 1) ? prs->charmaxlen :
1731 pg_mblen_range(prs->str + prs->state->posbyte,
1732 prs->str + prs->lenstr);
1733
1734 Assert(prs->state->posbyte + prs->state->charlen <= prs->lenstr);
1735 Assert(prs->state->state >= TPS_Base && prs->state->state < TPS_Null);
1736 Assert(Actions[prs->state->state].state == prs->state->state);
1737
1738 if (prs->state->pushedAtAction)
1739 {
1740 /* After a POP, pick up at the next test */
1741 item = prs->state->pushedAtAction + 1;
1742 prs->state->pushedAtAction = NULL;
1743 }
1744 else
1745 {
1746 item = Actions[prs->state->state].action;
1747 Assert(item != NULL);
1748 }
1749
1750 /* find action by character class */
1751 while (item->isclass)
1752 {
1753 prs->c = item->c;
1754 if (item->isclass(prs) != 0)
1755 break;
1756 item++;
1757 }
1758
1759 #ifdef WPARSER_TRACE
1760 {
1761 TParserPosition *ptr;
1762
1763 fprintf(stderr, "state ");
1764 /* indent according to stack depth */
1765 for (ptr = prs->state->prev; ptr; ptr = ptr->prev)
1766 fprintf(stderr, " ");
1767 fprintf(stderr, "%s ", Actions[prs->state->state].state_name);
1768 if (prs->state->posbyte < prs->lenstr)
1769 fprintf(stderr, "at %c", *(prs->str + prs->state->posbyte));
1770 else
1771 fprintf(stderr, "at EOF");
1772 fprintf(stderr, " matched rule %d flags%s%s%s%s%s%s%s%s%s%s%s\n",
1773 (int) (item - Actions[prs->state->state].action),
1774 (item->flags & A_BINGO) ? " BINGO" : "",
1775 (item->flags & A_POP) ? " POP" : "",
1776 (item->flags & A_PUSH) ? " PUSH" : "",
1777 (item->flags & A_RERUN) ? " RERUN" : "",
1778 (item->flags & A_CLEAR) ? " CLEAR" : "",
1779 (item->flags & A_MERGE) ? " MERGE" : "",
1780 (item->flags & A_CLRALL) ? " CLRALL" : "",
1781 (item->tostate != TPS_Null) ? " tostate " : "",
1782 (item->tostate != TPS_Null) ? Actions[item->tostate].state_name : "",
1783 (item->type > 0) ? " type " : "",
1784 tok_alias[item->type]);
1785 }
1786 #endif
1787
1788 /* call special handler if exists */
1789 if (item->special)
1790 item->special(prs);
1791
1792 /* BINGO, token is found */
1793 if (item->flags & A_BINGO)
1794 {
1795 Assert(item->type > 0);
1796 prs->lenbytetoken = prs->state->lenbytetoken;
1797 prs->lenchartoken = prs->state->lenchartoken;
1798 prs->state->lenbytetoken = prs->state->lenchartoken = 0;
1799 prs->type = item->type;
1800 }
1801
1802 /* do various actions by flags */
1803 if (item->flags & A_POP)
1804 { /* pop stored state in stack */
1805 TParserPosition *ptr = prs->state->prev;
1806
1807 pfree(prs->state);
1808 prs->state = ptr;
1809 Assert(prs->state);
1810 }
1811 else if (item->flags & A_PUSH)
1812 { /* push (store) state in stack */
1813 prs->state->pushedAtAction = item; /* remember where we push */
1814 prs->state = newTParserPosition(prs->state);
1815 }
1816 else if (item->flags & A_CLEAR)
1817 { /* clear previous pushed state */
1818 TParserPosition *ptr;
1819
1820 Assert(prs->state->prev);
1821 ptr = prs->state->prev->prev;
1822 pfree(prs->state->prev);
1823 prs->state->prev = ptr;
1824 }
1825 else if (item->flags & A_CLRALL)
1826 { /* clear all previous pushed state */
1827 TParserPosition *ptr;
1828
1829 while (prs->state->prev)
1830 {
1831 ptr = prs->state->prev->prev;
1832 pfree(prs->state->prev);
1833 prs->state->prev = ptr;
1834 }
1835 }
1836 else if (item->flags & A_MERGE)
1837 { /* merge posinfo with current and pushed state */
1838 TParserPosition *ptr = prs->state;
1839
1840 Assert(prs->state->prev);
1841 prs->state = prs->state->prev;
1842
1843 prs->state->posbyte = ptr->posbyte;
1844 prs->state->poschar = ptr->poschar;
1845 prs->state->charlen = ptr->charlen;
1846 prs->state->lenbytetoken = ptr->lenbytetoken;
1847 prs->state->lenchartoken = ptr->lenchartoken;
1848 pfree(ptr);
1849 }
1850
1851 /* set new state if pointed */
1852 if (item->tostate != TPS_Null)
1853 prs->state->state = item->tostate;
1854
1855 /* check for go away */
1856 if ((item->flags & A_BINGO) ||
1857 (prs->state->posbyte >= prs->lenstr &&
1858 (item->flags & A_RERUN) == 0))
1859 break;
1860
1861 /* go to beginning of loop if we should rerun or we just restore state */
1862 if (item->flags & (A_RERUN | A_POP))
1863 continue;
1864
1865 /* move forward */
1866 if (prs->state->charlen)
1867 {
1868 prs->state->posbyte += prs->state->charlen;
1869 prs->state->lenbytetoken += prs->state->charlen;
1870 prs->state->poschar++;
1871 prs->state->lenchartoken++;
1872 }
1873 }
1874
1875 return (item && (item->flags & A_BINGO));
1876 }
1877
1878 Datum
1879 prsd_lextype(PG_FUNCTION_ARGS)
1880 {
1881 LexDescr *descr = (LexDescr *) palloc(sizeof(LexDescr) * (LASTNUM + 1));
1882 int i;
1883
1884 for (i = 1; i <= LASTNUM; i++)
1885 {
1886 descr[i - 1].lexid = i;
1887 descr[i - 1].alias = pstrdup(tok_alias[i]);
1888 descr[i - 1].descr = pstrdup(lex_descr[i]);
1889 }
1890
1891 descr[LASTNUM].lexid = 0;
1892
1893 PG_RETURN_POINTER(descr);
1894 }
1895
1896 Datum
1897 prsd_start(PG_FUNCTION_ARGS)
1898 {
1899 PG_RETURN_POINTER(TParserInit((char *) PG_GETARG_POINTER(0), PG_GETARG_INT32(1)));
1900 }
1901
1902 Datum
1903 prsd_nexttoken(PG_FUNCTION_ARGS)
1904 {
1905 TParser *p = (TParser *) PG_GETARG_POINTER(0);
1906 char **t = (char **) PG_GETARG_POINTER(1);
1907 int *tlen = (int *) PG_GETARG_POINTER(2);
1908
1909 if (!TParserGet(p))
1910 PG_RETURN_INT32(0);
1911
1912 *t = p->token;
1913 *tlen = p->lenbytetoken;
1914
1915 PG_RETURN_INT32(p->type);
1916 }
1917
1918 Datum
1919 prsd_end(PG_FUNCTION_ARGS)
1920 {
1921 TParser *p = (TParser *) PG_GETARG_POINTER(0);
1922
1923 TParserClose(p);
1924 PG_RETURN_VOID();
1925 }
1926
1927
1928 /*
1929 * ts_headline support begins here
1930 */
1931
1932 /* token type classification macros */
1933 #define TS_IDIGNORE(x) ( (x)==TAG_T || (x)==PROTOCOL || (x)==SPACE || (x)==XMLENTITY )
1934 #define HLIDREPLACE(x) ( (x)==TAG_T )
1935 #define HLIDSKIP(x) ( (x)==URL_T || (x)==NUMHWORD || (x)==ASCIIHWORD || (x)==HWORD )
1936 #define XMLHLIDSKIP(x) ( (x)==URL_T || (x)==NUMHWORD || (x)==ASCIIHWORD || (x)==HWORD )
1937 #define NONWORDTOKEN(x) ( (x)==SPACE || HLIDREPLACE(x) || HLIDSKIP(x) )
1938 #define NOENDTOKEN(x) ( NONWORDTOKEN(x) || (x)==SCIENTIFIC || (x)==VERSIONNUMBER || (x)==DECIMAL_T || (x)==SIGNEDINT || (x)==UNSIGNEDINT || TS_IDIGNORE(x) )
1939
1940 /*
1941 * Macros useful in headline selection. These rely on availability of
1942 * "HeadlineParsedText *prs" describing some text, and "int shortword"
1943 * describing the "short word" length parameter.
1944 */
1945
1946 /* Interesting words are non-repeated search terms */
1947 #define INTERESTINGWORD(j) \
1948 (prs->words[j].item && !prs->words[j].repeated)
1949
1950 /* Don't want to end at a non-word or a short word, unless interesting */
1951 #define BADENDPOINT(j) \
1952 ((NOENDTOKEN(prs->words[j].type) || prs->words[j].len <= shortword) && \
1953 !INTERESTINGWORD(j))
1954
1955 typedef struct
1956 {
1957 /* one cover (well, really one fragment) for mark_hl_fragments */
1958 int32 startpos; /* fragment's starting word index */
1959 int32 endpos; /* ending word index (inclusive) */
1960 int32 poslen; /* number of interesting words */
1961 int32 curlen; /* total number of words */
1962 bool chosen; /* chosen? */
1963 bool excluded; /* excluded? */
1964 } CoverPos;
1965
1966 typedef struct
1967 {
1968 /* callback data for checkcondition_HL */
1969 HeadlineWordEntry *words;
1970 int len;
1971 } hlCheck;
1972
1973
1974 /*
1975 * TS_execute callback for matching a tsquery operand to headline words
1976 *
1977 * Note: it's tempting to report words[] indexes as pos values to save
1978 * searching in hlCover; but that would screw up phrase matching, which
1979 * expects to measure distances in lexemes not tokens.
1980 */
1981 static TSTernaryValue
1982 checkcondition_HL(void *opaque, QueryOperand *val, ExecPhraseData *data)
1983 {
1984 hlCheck *checkval = (hlCheck *) opaque;
1985 int i;
1986
1987 /* scan words array for matching items */
1988 for (i = 0; i < checkval->len; i++)
1989 {
1990 if (checkval->words[i].item == val)
1991 {
1992 /* if data == NULL, don't need to report positions */
1993 if (!data)
1994 return TS_YES;
1995
1996 if (!data->pos)
1997 {
1998 data->pos = palloc(sizeof(WordEntryPos) * checkval->len);
1999 data->allocated = true;
2000 data->npos = 1;
2001 data->pos[0] = checkval->words[i].pos;
2002 }
2003 else if (data->pos[data->npos - 1] < checkval->words[i].pos)
2004 {
2005 data->pos[data->npos++] = checkval->words[i].pos;
2006 }
2007 }
2008 }
2009
2010 if (data && data->npos > 0)
2011 return TS_YES;
2012
2013 return TS_NO;
2014 }
2015
2016 /*
2017 * hlCover: try to find a substring of prs' word list that satisfies query
2018 *
2019 * locations is the result of TS_execute_locations() for the query.
2020 * We use this to identify plausible subranges of the query.
2021 *
2022 * *nextpos is the lexeme position (NOT word index) to start the search
2023 * at. Caller should initialize this to zero. If successful, we'll
2024 * advance it to the next place to search at.
2025 *
2026 * On success, sets *p to first word index and *q to last word index of the
2027 * cover substring, and returns true.
2028 *
2029 * The result is a minimal cover, in the sense that both *p and *q will be
2030 * words used in the query.
2031 */
2032 static bool
2033 hlCover(HeadlineParsedText *prs, TSQuery query, List *locations,
2034 int *nextpos, int *p, int *q)
2035 {
2036 int pos = *nextpos;
2037
2038 /* This loop repeats when our selected word-range fails the query */
2039 for (;;)
2040 {
2041 int posb,
2042 pose;
2043 ListCell *lc;
2044
2045 /*
2046 * For each AND'ed query term or phrase, find its first occurrence at
2047 * or after pos; set pose to the maximum of those positions.
2048 *
2049 * We need not consider ORs or NOTs here; see the comments for
2050 * TS_execute_locations(). Rechecking the match with TS_execute(),
2051 * below, will deal with any ensuing imprecision.
2052 */
2053 pose = -1;
2054 foreach(lc, locations)
2055 {
2056 ExecPhraseData *pdata = (ExecPhraseData *) lfirst(lc);
2057 int first = -1;
2058
2059 for (int i = 0; i < pdata->npos; i++)
2060 {
2061 /* For phrase matches, use the ending lexeme */
2062 int endp = pdata->pos[i];
2063
2064 if (endp >= pos)
2065 {
2066 first = endp;
2067 break;
2068 }
2069 }
2070 if (first < 0)
2071 return false; /* no more matches for this term */
2072 if (first > pose)
2073 pose = first;
2074 }
2075
2076 if (pose < 0)
2077 return false; /* we only get here if empty list */
2078
2079 /*
2080 * Now, for each AND'ed query term or phrase, find its last occurrence
2081 * at or before pose; set posb to the minimum of those positions.
2082 *
2083 * We start posb at INT_MAX - 1 to guarantee no overflow if we compute
2084 * posb + 1 below.
2085 */
2086 posb = INT_MAX - 1;
2087 foreach(lc, locations)
2088 {
2089 ExecPhraseData *pdata = (ExecPhraseData *) lfirst(lc);
2090 int last = -1;
2091
2092 for (int i = pdata->npos - 1; i >= 0; i--)
2093 {
2094 /* For phrase matches, use the starting lexeme */
2095 int startp = pdata->pos[i] - pdata->width;
2096
2097 if (startp <= pose)
2098 {
2099 last = startp;
2100 break;
2101 }
2102 }
2103 if (last < posb)
2104 posb = last;
2105 }
2106
2107 /*
2108 * We could end up with posb to the left of pos, in case some phrase
2109 * match crosses pos. Try the match starting at pos anyway, since the
2110 * result of TS_execute_locations is imprecise for phrase matches OR'd
2111 * with plain matches; that is, if the query is "(A <-> B) | C" then C
2112 * could match at pos even though the phrase match would have to
2113 * extend to the left of pos.
2114 */
2115 posb = Max(posb, pos);
2116
2117 /* This test probably always succeeds, but be paranoid */
2118 if (posb <= pose)
2119 {
2120 /*
2121 * posb .. pose is now the shortest, earliest-after-pos range of
2122 * lexeme positions containing all the query terms. It will
2123 * contain all phrase matches, too, except in the corner case
2124 * described just above.
2125 *
2126 * Now convert these lexeme positions to indexes in prs->words[].
2127 */
2128 int idxb = -1;
2129 int idxe = -1;
2130
2131 for (int i = 0; i < prs->curwords; i++)
2132 {
2133 if (prs->words[i].item == NULL)
2134 continue;
2135 if (idxb < 0 && prs->words[i].pos >= posb)
2136 idxb = i;
2137 if (prs->words[i].pos <= pose)
2138 idxe = i;
2139 else
2140 break;
2141 }
2142
2143 /* This test probably always succeeds, but be paranoid */
2144 if (idxb >= 0 && idxe >= idxb)
2145 {
2146 /*
2147 * Finally, check that the selected range satisfies the query.
2148 * This should succeed in all simple cases; but odd cases
2149 * involving non-top-level NOT conditions or phrase matches
2150 * OR'd with other things could fail, since the result of
2151 * TS_execute_locations doesn't fully represent such things.
2152 */
2153 hlCheck ch;
2154
2155 ch.words = &(prs->words[idxb]);
2156 ch.len = idxe - idxb + 1;
2157 if (TS_execute(GETQUERY(query), &ch,
2158 TS_EXEC_EMPTY, checkcondition_HL))
2159 {
2160 /* Match! Advance *nextpos and return the word range. */
2161 *nextpos = posb + 1;
2162 *p = idxb;
2163 *q = idxe;
2164 return true;
2165 }
2166 }
2167 }
2168
2169 /*
2170 * Advance pos and try again. Any later workable match must start
2171 * beyond posb.
2172 */
2173 pos = posb + 1;
2174 }
2175 /* Can't get here, but stupider compilers complain if we leave it off */
2176 return false;
2177 }
2178
2179 /*
2180 * Apply suitable highlight marking to words selected by headline selector
2181 *
2182 * The words from startpos to endpos inclusive are marked per highlightall
2183 */
2184 static void
2185 mark_fragment(HeadlineParsedText *prs, bool highlightall,
2186 int startpos, int endpos)
2187 {
2188 int i;
2189
2190 for (i = startpos; i <= endpos; i++)
2191 {
2192 if (prs->words[i].item)
2193 prs->words[i].selected = 1;
2194 if (!highlightall)
2195 {
2196 if (HLIDREPLACE(prs->words[i].type))
2197 prs->words[i].replace = 1;
2198 else if (HLIDSKIP(prs->words[i].type))
2199 prs->words[i].skip = 1;
2200 }
2201 else
2202 {
2203 if (XMLHLIDSKIP(prs->words[i].type))
2204 prs->words[i].skip = 1;
2205 }
2206
2207 prs->words[i].in = (prs->words[i].repeated) ? 0 : 1;
2208 }
2209 }
2210
2211 /*
2212 * split a cover substring into fragments not longer than max_words
2213 *
2214 * At entry, *startpos and *endpos are the (remaining) bounds of the cover
2215 * substring. They are updated to hold the bounds of the next fragment.
2216 *
2217 * *curlen and *poslen are set to the fragment's length, in words and
2218 * interesting words respectively.
2219 */
2220 static void
2221 get_next_fragment(HeadlineParsedText *prs, int *startpos, int *endpos,
2222 int *curlen, int *poslen, int max_words)
2223 {
2224 int i;
2225
2226 /*
2227 * Objective: select a fragment of words between startpos and endpos such
2228 * that it has at most max_words and both ends have query words. If the
2229 * startpos and endpos are the endpoints of the cover and the cover has
2230 * fewer words than max_words, then this function should just return the
2231 * cover
2232 */
2233 /* first move startpos to an item */
2234 for (i = *startpos; i <= *endpos; i++)
2235 {
2236 *startpos = i;
2237 if (INTERESTINGWORD(i))
2238 break;
2239 }
2240 /* cut endpos to have only max_words */
2241 *curlen = 0;
2242 *poslen = 0;
2243 for (i = *startpos; i <= *endpos && *curlen < max_words; i++)
2244 {
2245 if (!NONWORDTOKEN(prs->words[i].type))
2246 *curlen += 1;
2247 if (INTERESTINGWORD(i))
2248 *poslen += 1;
2249 }
2250 /* if the cover was cut then move back endpos to a query item */
2251 if (*endpos > i)
2252 {
2253 *endpos = i;
2254 for (i = *endpos; i >= *startpos; i--)
2255 {
2256 *endpos = i;
2257 if (INTERESTINGWORD(i))
2258 break;
2259 if (!NONWORDTOKEN(prs->words[i].type))
2260 *curlen -= 1;
2261 }
2262 }
2263 }
2264
2265 /*
2266 * Headline selector used when MaxFragments > 0
2267 *
2268 * Note: in this mode, highlightall is disregarded for phrase selection;
2269 * it only controls presentation details.
2270 */
2271 static void
2272 mark_hl_fragments(HeadlineParsedText *prs, TSQuery query, List *locations,
2273 bool highlightall,
2274 int shortword, int min_words,
2275 int max_words, int max_fragments)
2276 {
2277 int32 poslen,
2278 curlen,
2279 i,
2280 f,
2281 num_f = 0;
2282 int32 stretch,
2283 maxstretch,
2284 posmarker;
2285
2286 int32 startpos = 0,
2287 endpos = 0,
2288 nextpos = 0,
2289 p = 0,
2290 q = 0;
2291
2292 int32 numcovers = 0,
2293 maxcovers = 32;
2294
2295 int32 minI,
2296 minwords,
2297 maxitems;
2298 CoverPos *covers;
2299
2300 covers = palloc(maxcovers * sizeof(CoverPos));
2301
2302 /* get all covers */
2303 while (hlCover(prs, query, locations, &nextpos, &p, &q))
2304 {
2305 startpos = p;
2306 endpos = q;
2307
2308 /*
2309 * Break the cover into smaller fragments such that each fragment has
2310 * at most max_words. Also ensure that each end of each fragment is a
2311 * query word. This will allow us to stretch the fragment in either
2312 * direction
2313 */
2314
2315 while (startpos <= endpos)
2316 {
2317 get_next_fragment(prs, &startpos, &endpos, &curlen, &poslen, max_words);
2318 if (numcovers >= maxcovers)
2319 {
2320 maxcovers *= 2;
2321 covers = repalloc(covers, sizeof(CoverPos) * maxcovers);
2322 }
2323 covers[numcovers].startpos = startpos;
2324 covers[numcovers].endpos = endpos;
2325 covers[numcovers].curlen = curlen;
2326 covers[numcovers].poslen = poslen;
2327 covers[numcovers].chosen = false;
2328 covers[numcovers].excluded = false;
2329 numcovers++;
2330 startpos = endpos + 1;
2331 endpos = q;
2332 }
2333 }
2334
2335 /* choose best covers */
2336 for (f = 0; f < max_fragments; f++)
2337 {
2338 maxitems = 0;
2339 minwords = PG_INT32_MAX;
2340 minI = -1;
2341
2342 /*
2343 * Choose the cover that contains max items. In case of tie choose the
2344 * one with smaller number of words.
2345 */
2346 for (i = 0; i < numcovers; i++)
2347 {
2348 if (!covers[i].chosen && !covers[i].excluded &&
2349 (maxitems < covers[i].poslen ||
2350 (maxitems == covers[i].poslen &&
2351 minwords > covers[i].curlen)))
2352 {
2353 maxitems = covers[i].poslen;
2354 minwords = covers[i].curlen;
2355 minI = i;
2356 }
2357 }
2358 /* if a cover was found mark it */
2359 if (minI >= 0)
2360 {
2361 covers[minI].chosen = true;
2362 /* adjust the size of cover */
2363 startpos = covers[minI].startpos;
2364 endpos = covers[minI].endpos;
2365 curlen = covers[minI].curlen;
2366 /* stretch the cover if cover size is lower than max_words */
2367 if (curlen < max_words)
2368 {
2369 /* divide the stretch on both sides of cover */
2370 maxstretch = (max_words - curlen) / 2;
2371
2372 /*
2373 * first stretch the startpos stop stretching if 1. we hit the
2374 * beginning of document 2. exceed maxstretch 3. we hit an
2375 * already marked fragment
2376 */
2377 stretch = 0;
2378 posmarker = startpos;
2379 for (i = startpos - 1; i >= 0 && stretch < maxstretch && !prs->words[i].in; i--)
2380 {
2381 if (!NONWORDTOKEN(prs->words[i].type))
2382 {
2383 curlen++;
2384 stretch++;
2385 }
2386 posmarker = i;
2387 }
2388 /* cut back startpos till we find a good endpoint */
2389 for (i = posmarker; i < startpos && BADENDPOINT(i); i++)
2390 {
2391 if (!NONWORDTOKEN(prs->words[i].type))
2392 curlen--;
2393 }
2394 startpos = i;
2395 /* now stretch the endpos as much as possible */
2396 posmarker = endpos;
2397 for (i = endpos + 1; i < prs->curwords && curlen < max_words && !prs->words[i].in; i++)
2398 {
2399 if (!NONWORDTOKEN(prs->words[i].type))
2400 curlen++;
2401 posmarker = i;
2402 }
2403 /* cut back endpos till we find a good endpoint */
2404 for (i = posmarker; i > endpos && BADENDPOINT(i); i--)
2405 {
2406 if (!NONWORDTOKEN(prs->words[i].type))
2407 curlen--;
2408 }
2409 endpos = i;
2410 }
2411 covers[minI].startpos = startpos;
2412 covers[minI].endpos = endpos;
2413 covers[minI].curlen = curlen;
2414 /* Mark the chosen fragments (covers) */
2415 mark_fragment(prs, highlightall, startpos, endpos);
2416 num_f++;
2417 /* Exclude covers overlapping this one from future consideration */
2418 for (i = 0; i < numcovers; i++)
2419 {
2420 if (i != minI &&
2421 ((covers[i].startpos >= startpos &&
2422 covers[i].startpos <= endpos) ||
2423 (covers[i].endpos >= startpos &&
2424 covers[i].endpos <= endpos) ||
2425 (covers[i].startpos < startpos &&
2426 covers[i].endpos > endpos)))
2427 covers[i].excluded = true;
2428 }
2429 }
2430 else
2431 break; /* no selectable covers remain */
2432 }
2433
2434 /* show the first min_words words if we have not marked anything */
2435 if (num_f <= 0)
2436 {
2437 startpos = curlen = 0;
2438 endpos = -1;
2439 for (i = 0; i < prs->curwords && curlen < min_words; i++)
2440 {
2441 if (!NONWORDTOKEN(prs->words[i].type))
2442 curlen++;
2443 endpos = i;
2444 }
2445 mark_fragment(prs, highlightall, startpos, endpos);
2446 }
2447
2448 pfree(covers);
2449 }
2450
2451 /*
2452 * Headline selector used when MaxFragments == 0
2453 */
2454 static void
2455 mark_hl_words(HeadlineParsedText *prs, TSQuery query, List *locations,
2456 bool highlightall,
2457 int shortword, int min_words, int max_words)
2458 {
2459 int nextpos = 0,
2460 p = 0,
2461 q = 0;
2462 int bestb = -1,
2463 beste = -1;
2464 int bestlen = -1;
2465 bool bestcover = false;
2466 int pose,
2467 posb,
2468 poslen,
2469 curlen;
2470 bool poscover;
2471 int i;
2472
2473 if (!highlightall)
2474 {
2475 /* examine all covers, select a headline using the best one */
2476 while (hlCover(prs, query, locations, &nextpos, &p, &q))
2477 {
2478 /*
2479 * Count words (curlen) and interesting words (poslen) within
2480 * cover, but stop once we reach max_words. This step doesn't
2481 * consider whether that's a good stopping point. posb and pose
2482 * are set to the start and end indexes of the possible headline.
2483 */
2484 curlen = 0;
2485 poslen = 0;
2486 posb = pose = p;
2487 for (i = p; i <= q && curlen < max_words; i++)
2488 {
2489 if (!NONWORDTOKEN(prs->words[i].type))
2490 curlen++;
2491 if (INTERESTINGWORD(i))
2492 poslen++;
2493 pose = i;
2494 }
2495
2496 if (curlen < max_words)
2497 {
2498 /*
2499 * We have room to lengthen the headline, so search forward
2500 * until it's full or we find a good stopping point. We'll
2501 * reconsider the word at "q", then move forward.
2502 */
2503 for (i = i - 1; i < prs->curwords && curlen < max_words; i++)
2504 {
2505 if (i > q)
2506 {
2507 if (!NONWORDTOKEN(prs->words[i].type))
2508 curlen++;
2509 if (INTERESTINGWORD(i))
2510 poslen++;
2511 }
2512 pose = i;
2513 if (BADENDPOINT(i))
2514 continue;
2515 if (curlen >= min_words)
2516 break;
2517 }
2518 if (curlen < min_words)
2519 {
2520 /*
2521 * Reached end of text and our headline is still shorter
2522 * than min_words, so try to extend it to the left.
2523 */
2524 for (i = p - 1; i >= 0; i--)
2525 {
2526 if (!NONWORDTOKEN(prs->words[i].type))
2527 curlen++;
2528 if (INTERESTINGWORD(i))
2529 poslen++;
2530 if (curlen >= max_words)
2531 break;
2532 if (BADENDPOINT(i))
2533 continue;
2534 if (curlen >= min_words)
2535 break;
2536 }
2537 posb = (i >= 0) ? i : 0;
2538 }
2539 }
2540 else
2541 {
2542 /*
2543 * Can't make headline longer, so consider making it shorter
2544 * if needed to avoid a bad endpoint.
2545 */
2546 if (i > q)
2547 i = q;
2548 for (; curlen > min_words; i--)
2549 {
2550 if (!BADENDPOINT(i))
2551 break;
2552 if (!NONWORDTOKEN(prs->words[i].type))
2553 curlen--;
2554 if (INTERESTINGWORD(i))
2555 poslen--;
2556 pose = i - 1;
2557 }
2558 }
2559
2560 /*
2561 * Check whether the proposed headline includes the original
2562 * cover; it might not if we trimmed it due to max_words.
2563 */
2564 poscover = (posb <= p && pose >= q);
2565
2566 /*
2567 * Adopt this headline if it's better than the last one, giving
2568 * highest priority to headlines including the cover, then to
2569 * headlines with more interesting words, then to headlines with
2570 * good stopping points. (Since bestlen is initially -1, we will
2571 * certainly adopt the first headline.)
2572 */
2573 if (poscover > bestcover ||
2574 (poscover == bestcover && poslen > bestlen) ||
2575 (poscover == bestcover && poslen == bestlen &&
2576 !BADENDPOINT(pose) && BADENDPOINT(beste)))
2577 {
2578 bestb = posb;
2579 beste = pose;
2580 bestlen = poslen;
2581 bestcover = poscover;
2582 }
2583 }
2584
2585 /*
2586 * If we found nothing acceptable, select min_words words starting at
2587 * the beginning.
2588 */
2589 if (bestlen < 0)
2590 {
2591 curlen = 0;
2592 pose = -1;
2593 for (i = 0; i < prs->curwords && curlen < min_words; i++)
2594 {
2595 if (!NONWORDTOKEN(prs->words[i].type))
2596 curlen++;
2597 pose = i;
2598 }
2599 bestb = 0;
2600 beste = pose;
2601 }
2602 }
2603 else
2604 {
2605 /* highlightall mode: headline is whole document */
2606 bestb = 0;
2607 beste = prs->curwords - 1;
2608 }
2609
2610 mark_fragment(prs, highlightall, bestb, beste);
2611 }
2612
2613 /*
2614 * Default parser's prsheadline function
2615 */
2616 Datum
2617 prsd_headline(PG_FUNCTION_ARGS)
2618 {
2619 HeadlineParsedText *prs = (HeadlineParsedText *) PG_GETARG_POINTER(0);
2620 List *prsoptions = (List *) PG_GETARG_POINTER(1);
2621 TSQuery query = PG_GETARG_TSQUERY(2);
2622 List *locations;
2623
2624 /* default option values: */
2625 int min_words = 15;
2626 int max_words = 35;
2627 int shortword = 3;
2628 int max_fragments = 0;
2629 bool highlightall = false;
2630 ListCell *l;
2631 size_t startsellen;
2632 size_t stopsellen;
2633 size_t fragdelimlen;
2634
2635 /* Extract configuration option values */
2636 prs->startsel = NULL;
2637 prs->stopsel = NULL;
2638 prs->fragdelim = NULL;
2639 foreach(l, prsoptions)
2640 {
2641 DefElem *defel = (DefElem *) lfirst(l);
2642 char *val = defGetString(defel);
2643
2644 if (pg_strcasecmp(defel->defname, "MaxWords") == 0)
2645 max_words = pg_strtoint32(val);
2646 else if (pg_strcasecmp(defel->defname, "MinWords") == 0)
2647 min_words = pg_strtoint32(val);
2648 else if (pg_strcasecmp(defel->defname, "ShortWord") == 0)
2649 shortword = pg_strtoint32(val);
2650 else if (pg_strcasecmp(defel->defname, "MaxFragments") == 0)
2651 max_fragments = pg_strtoint32(val);
2652 else if (pg_strcasecmp(defel->defname, "StartSel") == 0)
2653 prs->startsel = pstrdup(val);
2654 else if (pg_strcasecmp(defel->defname, "StopSel") == 0)
2655 prs->stopsel = pstrdup(val);
2656 else if (pg_strcasecmp(defel->defname, "FragmentDelimiter") == 0)
2657 prs->fragdelim = pstrdup(val);
2658 else if (pg_strcasecmp(defel->defname, "HighlightAll") == 0)
2659 highlightall = (pg_strcasecmp(val, "1") == 0 ||
2660 pg_strcasecmp(val, "on") == 0 ||
2661 pg_strcasecmp(val, "true") == 0 ||
2662 pg_strcasecmp(val, "t") == 0 ||
2663 pg_strcasecmp(val, "y") == 0 ||
2664 pg_strcasecmp(val, "yes") == 0);
2665 else
2666 ereport(ERROR,
2667 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
2668 errmsg("unrecognized headline parameter: \"%s\"",
2669 defel->defname)));
2670 }
2671
2672 /* in HighlightAll mode these parameters are ignored */
2673 if (!highlightall)
2674 {
2675 if (min_words >= max_words)
2676 ereport(ERROR,
2677 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
2678 errmsg("%s must be less than %s", "MinWords", "MaxWords")));
2679 if (min_words <= 0)
2680 ereport(ERROR,
2681 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
2682 errmsg("%s must be positive", "MinWords")));
2683 if (shortword < 0)
2684 ereport(ERROR,
2685 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
2686 errmsg("%s must be >= 0", "ShortWord")));
2687 if (max_fragments < 0)
2688 ereport(ERROR,
2689 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
2690 errmsg("%s must be >= 0", "MaxFragments")));
2691 }
2692
2693 /* Locate words and phrases matching the query */
2694 if (query->size > 0)
2695 {
2696 hlCheck ch;
2697
2698 ch.words = prs->words;
2699 ch.len = prs->curwords;
2700 locations = TS_execute_locations(GETQUERY(query), &ch, TS_EXEC_EMPTY,
2701 checkcondition_HL);
2702 }
2703 else
2704 locations = NIL; /* empty query matches nothing */
2705
2706 /* Apply appropriate headline selector */
2707 if (max_fragments == 0)
2708 mark_hl_words(prs, query, locations, highlightall, shortword,
2709 min_words, max_words);
2710 else
2711 mark_hl_fragments(prs, query, locations, highlightall, shortword,
2712 min_words, max_words, max_fragments);
2713
2714 /* Fill in default values for string options */
2715 if (!prs->startsel)
2716 prs->startsel = pstrdup("<b>");
2717 if (!prs->stopsel)
2718 prs->stopsel = pstrdup("</b>");
2719 if (!prs->fragdelim)
2720 prs->fragdelim = pstrdup(" ... ");
2721
2722 /* Caller will need these lengths, too */
2723 startsellen = strlen(prs->startsel);
2724 stopsellen = strlen(prs->stopsel);
2725 fragdelimlen = strlen(prs->fragdelim);
2726 if (startsellen > PG_INT16_MAX)
2727 ereport(ERROR,
2728 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
2729 errmsg("value for \"%s\" is too long", "StartSel")));
2730 if (stopsellen > PG_INT16_MAX)
2731 ereport(ERROR,
2732 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
2733 errmsg("value for \"%s\" is too long", "StopSel")));
2734 if (fragdelimlen > PG_INT16_MAX)
2735 ereport(ERROR,
2736 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
2737 errmsg("value for \"%s\" is too long", "FragmentDelimiter")));
2738 prs->startsellen = startsellen;
2739 prs->stopsellen = stopsellen;
2740 prs->fragdelimlen = fragdelimlen;
2741
2742 PG_RETURN_POINTER(prs);
2743 }