Debian Codegraph @

modules/gcide/markup.l

raw
1 %top {
2 /* This file is part of GNU Dico.
3 Copyright (C) 2012-2024 Sergey Poznyakoff
4
5 GNU Dico is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 3, or (at your option)
8 any later version.
9
10 GNU Dico is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with GNU Dico. If not, see <http://www.gnu.org/licenses/>. */
17
18 #include <config.h>
19 #include <dico.h>
20 #include <unistd.h>
21 #include <getopt.h>
22 #include <stdio.h>
23 #include <stdlib.h>
24 #include <errno.h>
25 #include <sysexits.h>
26 #include <setjmp.h>
27 #include <ctype.h>
28 #include <appi18n.h>
29 #include "gcide.h"
30 #include "wordsplit.h"
31
32 #define yy_create_buffer gcide_markup_yy_create_buffer
33 #define yy_delete_buffer gcide_markup_yy_delete_buffer
34 #define yy_flex_debug gcide_markup_yy_flex_debug
35 #define yy_init_buffer gcide_markup_yy_init_buffer
36 #define yy_flush_buffer gcide_markup_yy_flush_buffer
37 #define yy_load_buffer_state gcide_markup_yy_load_buffer_state
38 #define yy_switch_to_buffer gcide_markup_yy_switch_to_buffer
39 #define yyin gcide_markup_yyin
40 #define yyleng gcide_markup_yyleng
41 #define yylex gcide_markup_yylex
42 #define yylineno gcide_markup_yylineno
43 #define yyout gcide_markup_yyout
44 #define yyrestart gcide_markup_yyrestart
45 #define yytext gcide_markup_yytext
46 #define yywrap gcide_markup_yywrap
47 #define yyalloc gcide_markup_yyalloc
48 #define yyrealloc gcide_markup_yyrealloc
49 #define yyfree gcide_markup_yyfree
50 #define yyunput gcide_markup_yyunput
51
52 #define yylex_destroy gcide_markup_yylex_destroy
53 #define yyget_debug gcide_markup_yyget_debug
54 #define yyset_debug gcide_markup_yyset_debug
55 #define yyget_extra gcide_markup_yyget_extra
56 #define yyset_extra gcide_markup_yyset_extra
57 #define yyget_in gcide_markup_yyget_in
58 #define yyset_in gcide_markup_yyset_in
59 #define yyget_out gcide_markup_yyget_out
60 #define yyset_out gcide_markup_yyset_out
61 #define yyget_leng gcide_markup_yyget_leng
62 #define yyget_text gcide_markup_yyget_text
63 #define yyget_lineno gcide_markup_yyget_lineno
64 #define yyset_lineno gcide_markup_yyset_lineno
65
66 }
67 %{
68
69 static struct gcide_locus *base_locus;
70 static char const *input_buf;
71 static size_t input_len;
72 static unsigned token_beg;
73 static unsigned token_end;
74
75 /*
76 * Dictionary entries in GCIDE corpus files begin with "<p><ent>", but index
77 * entries in the GCIDE.IDX file point to the first "<hw>" tag that follows.
78 * Thus, the very first "</p>" encountered when parsing an entry has no
79 * corresponding "<p>" tag. The first_p_seen variable is set by push_tag
80 * when it sees the first opening "<p>" tag and is used in pop_tag to generate
81 * the first "p" block tag and avoid issuing a spurious error message when
82 * the first "</p>" is encountered.
83 */
84 static int first_p_seen;
85
86 #define LOCUS_FILE (base_locus ? base_locus->file : "")
87 #define LOCUS_OFF(off) ((base_locus ? base_locus->offset : 0) + off)
88
89 static char *textspace; /* Text storage space */
90 static size_t textsize; /* Size of text space */
91 static size_t textpos; /* Current position in the text space */
92 static size_t textstart; /* Start of the current text segment */
93 static size_t textstart_offset;
94
95 static dico_list_t tagstk;
96 static struct gcide_tag *current_tag;
97
98 static jmp_buf errbuf;
99
100 static void
101 memerr(const char *text)
102 {
103 dico_log(L_ERR, ENOMEM, "%s", text);
104 longjmp(errbuf, 1);
105 }
106
107 #define YY_USER_ACTION do { \
108 token_beg = token_end; \
109 token_end += yyleng; \
110 } while (0);
111 #define YY_INPUT(buf,result,max_size) do { \
112 size_t __n = (max_size) > input_len ? input_len : (max_size); \
113 if (__n) \
114 memcpy((buf), input_buf, __n); \
115 input_buf += __n; \
116 input_len -= __n; \
117 (result) = __n; \
118 } while(0)
119
120 static int retstate;
121 static char *endtag;
122 static size_t endtaglen;
123 #define BEGIN_COMMENT(end) \
124 { retstate = YYSTATE; endtag = (end); endtaglen = strlen(endtag); BEGIN(COMMENT); }
125
126 static void
127 text_assert_size(size_t len)
128 {
129 while (textsize - textpos < len) {
130 size_t nsize;
131 char *newp;
132
133 if (SIZE_MAX / 2 < textsize)
134 memerr("text_add_str");
135 nsize = 2 * textsize;
136 newp = realloc(textspace, nsize);
137 if (!newp)
138 memerr("text_add_str");
139 textspace = newp;
140 textsize = nsize;
141 }
142 }
143
144 static void
145 text_add_str(char const *s, size_t l)
146 {
147 if (textstart == textpos) textstart_offset = token_beg;
148 text_assert_size(l);
149 memcpy(textspace + textpos, s, l);
150 textpos += l;
151 }
152
153 static void
154 text_add_strz(char const *s)
155 {
156 text_add_str(s, strlen(s));
157 }
158
159 static void
160 text_add_chr(int ch)
161 {
162 char c = ch;
163 text_add_str(&c, 1);
164 }
165
166 #define text_segment_length() (textpos - textstart)
167
168 static size_t
169 text_segment_finish()
170 {
171 size_t ret = textstart;
172 text_add_chr(0);
173 textstart = textpos;
174 return ret;
175 }
176
177 static int free_tag(void *item, void *data);
178
179 static struct gcide_tag *
180 alloc_tag(enum gcide_content_type type)
181 {
182 dico_list_t list;
183 struct gcide_tag *tag = calloc(1, sizeof(*tag));
184 if (!tag)
185 memerr("alloc_tag");
186 tag->tag_type = type;
187 switch (type) {
188 case gcide_content_top:
189 case gcide_content_tag:
190 list = dico_list_create();
191 if (!list) {
192 free(tag);
193 memerr("alloc_tag");
194 }
195 dico_list_set_free_item(list, free_tag, NULL);
196 tag->v.tag.taglist = list;
197 break;
198 default:
199 break;
200 }
201 return tag;
202 }
203
204 struct gcide_tag *
205 gcide_tag_alloc(const char *text, size_t len)
206 {
207 struct gcide_tag *tag;
208 struct wordsplit ws;
209 size_t i;
210
211 if (wordsplit_len(text, len, &ws, WRDSF_DEFFLAGS & ~WRDSF_CESCAPES)) {
212 dico_log(L_ERR, 0, _("cannot parse line %.*s: %s"),
213 (int)len, text, wordsplit_strerror(&ws));
214 longjmp(errbuf, 1);
215 }
216
217 for (i = 0; i < ws.ws_wordc; i++) {
218 char *p;
219 for (p = ws.ws_wordv[i]; *p; p++) {
220 if (*p == '=')
221 break;
222 *p = tolower(*p);
223 }
224 }
225 tag = alloc_tag(gcide_content_tag);
226 tag->v.tag.tag_parmc = ws.ws_wordc;
227 tag->v.tag.tag_parmv = ws.ws_wordv;
228 tag->offset = token_beg;
229 ws.ws_wordc = 0;
230 ws.ws_wordv = NULL;
231 wordsplit_free(&ws);
232 return tag;
233 }
234
235 #define alloc_taglist gcide_tag_alloc
236
237 void
238 gcide_tag_free(struct gcide_tag *tag)
239 {
240 if (tag) {
241 switch (tag->tag_type) {
242 default:
243 break;
244 case gcide_content_top:
245 case gcide_content_tag:
246 dico_list_destroy(&tag->v.tag.taglist);
247 }
248 free(tag);
249 }
250 }
251
252 int
253 gcide_is_tag(struct gcide_tag *tag, char const *name)
254 {
255 return tag != NULL && tag->tag_type == gcide_content_tag &&
256 strcmp(tag->tag_name, name) == 0;
257 }
258
259 static int
260 free_tag(void *item, void *data)
261 {
262 gcide_tag_free(item);
263 return 0;
264 }
265
266 static void
267 append_tag(struct gcide_tag *tag)
268 {
269 dico_list_t list;
270
271 switch (current_tag->tag_type) {
272 case gcide_content_text: {
273 struct gcide_tag *subtag = alloc_tag(gcide_content_text);
274 subtag->offset = current_tag->offset;
275 subtag->v.textpos = current_tag->v.textpos;
276 list = dico_list_create();
277 if (!list) {
278 free(subtag);
279 free(tag);
280 memerr("append_tag");
281 }
282 dico_list_set_free_item(list, free_tag, NULL);
283 dico_list_append(list, subtag);
284 current_tag->tag_type = gcide_content_tag;
285 current_tag->v.tag.taglist = list;
286 break;
287 }
288
289 default:
290 break;
291 }
292
293 dico_list_append(current_tag->v.tag.taglist, tag);
294 }
295
296 static void push_tag(struct gcide_tag *tag);
297 static void pop_tag(const char *tagstr, size_t taglen);
298
299 static void
300 flush_text_segment(void)
301 {
302 if (text_segment_length()) {
303 struct gcide_tag *tag = alloc_tag(gcide_content_text);
304 tag->v.textpos = text_segment_finish();
305 tag->offset = textstart_offset;
306 append_tag(tag);
307 }
308 }
309 %}
310
311 %option 8bit
312 %option nounput
313 %option noinput
314 %option nodefault
315
316 %x COMMENT PR HW PRE
317
318 XD [0-9a-f]
319 WS [ \t]
320 %%
321 <INITIAL,PR,HW,PRE>{
322 "<--" BEGIN_COMMENT("-->");
323 "<!" BEGIN_COMMENT("!>");
324 "<"[a-zA-Z][a-zA-Z0-9._-]*">" {
325 flush_text_segment();
326 push_tag(alloc_taglist(yytext + 1, yyleng - 2));
327 }
328 "<"[a-zA-Z][a-zA-Z0-9._-]*{WS}[^>]*">" {
329 flush_text_segment();
330 push_tag(alloc_taglist(yytext + 1, yyleng - 2));
331 }
332 "</"[a-zA-Z][a-zA-Z0-9._-]*">" {
333 flush_text_segment();
334 pop_tag(yytext + 2, yyleng - 3);
335 }
336 "</"[a-zA-Z][^>]*"/" {
337 dico_log(L_WARN, 0, _("%s:%zu: malformed entity or tag: %s"),
338 LOCUS_FILE, LOCUS_OFF(token_beg), yytext);
339 }
340 "<br/" {
341 flush_text_segment();
342 append_tag(alloc_tag(gcide_content_br));
343 }
344 "<"[a-zA-Z?][a-zA-Z0-9]*"/" {
345 char const *s = gcide_entity_to_utf8(yytext);
346 if (s)
347 text_add_str(s, strlen(s));
348 else
349 dico_log(L_WARN, 0, _("%s:%zu: unrecognized entity: %s"),
350 LOCUS_FILE, LOCUS_OFF(token_beg), yytext);
351 }
352 "\\'"{XD}{XD} {
353 char const *s = gcide_escape_to_utf8(yytext+2);
354
355 if (s)
356 text_add_str(s, strlen(s));
357 else {
358 text_add_str(yytext, yyleng);
359 dico_log(L_WARN, 0,
360 _("%s:%zu: unknown character sequence %s"),
361 LOCUS_FILE, LOCUS_OFF(token_beg), yytext);
362 }
363 }
364 [<\\](.|\n) {
365 text_add_chr(yytext[0]);
366 yyless(1);
367 token_end--;
368 }
369 \r ;
370 }
371 <INITIAL,PR,HW>{
372 \n+ {
373 flush_text_segment();
374 append_tag(alloc_tag(gcide_content_nl));
375 }
376 }
377 <INITIAL>{
378 [^<\\\r\n]+ text_add_str(yytext, yyleng);
379 }
380 <PRE>{
381 [^<\\\r]+ text_add_str(yytext, yyleng);
382 }
383 <HW>{
384 [""`*] ;
385 [^<\\\r""`*]+ text_add_str(yytext, yyleng);
386 }
387 <PR>{
388 "\"" text_add_strz("ˈ"); /* Main accent */
389 "`" text_add_strz("ˌ"); /* Secondary accent */
390 "*" text_add_strz("‑"); /* Syllable break */
391 [^<\\\r""`*]+ text_add_str(yytext, yyleng);
392 }
393 <COMMENT>{
394 [^>]+">" {
395 if (yyleng >= endtaglen &&
396 memcmp(yytext + yyleng - endtaglen, endtag, endtaglen) == 0)
397 BEGIN(retstate);
398 }
399 .|\n ;
400 }
401
402 <INITIAL,PR,HW,PRE>. {
403 if (input_len > 0)
404 dico_log(L_NOTICE, 0,
405 _("%s:%zu: unmatched character %03o; please report"),
406 LOCUS_FILE, LOCUS_OFF(token_beg), yytext[0]);
407 text_add_chr(yytext[0]);
408 }
409 %%
410
411 int
412 yywrap()
413 {
414 return 1;
415 }
416
417 static void
418 push_tag(struct gcide_tag *tag)
419 {
420 append_tag(tag);
421 dico_list_push(tagstk, current_tag);
422 current_tag = tag;
423 if (strcmp(current_tag->tag_name, "pr") == 0)
424 BEGIN(PR);
425 else if (strcmp(current_tag->tag_name, "hw") == 0)
426 BEGIN(HW);
427 else if (strcmp(current_tag->tag_name, "pre") == 0)
428 BEGIN(PRE);
429 else if (strcmp(current_tag->tag_name, "p") == 0)
430 first_p_seen = 1;
431 }
432
433 static void
434 pop_tag(const char *tagstr, size_t taglen)
435 {
436 int len;
437 char *p;
438
439 text_assert_size(taglen);
440 p = textspace + textpos;
441 for (len = 0; len < taglen; len++) {
442 if (tagstr[len] == ' ' || tagstr[len] == '\t')
443 break;
444 *p++ = tolower(tagstr[len]);
445 }
446 tagstr = textspace + textpos;
447
448 if ((len == 2 && (memcmp(tagstr, "pr", len) == 0 ||
449 memcmp(tagstr, "hw", len) == 0)) ||
450 (len == 3 && memcmp(tagstr, "pre", len) == 0))
451 BEGIN(INITIAL);
452
453 if (current_tag->tag_type == gcide_content_tag &&
454 strlen(current_tag->tag_name) == len &&
455 memcmp(current_tag->tag_name, tagstr, len) == 0)
456 current_tag = dico_list_pop(tagstk);
457 else if (current_tag->tag_type == gcide_content_top &&
458 len == 1 &&
459 tagstr[0] == 'p' &&
460 !first_p_seen) {
461 /*
462 * The first instance of </p> has no corresponding open <p> tag
463 * because idxgcide stores the offset of the first <hw> tag in the
464 * index entry, whereas the opening <p> tag in GCIDE entries appears
465 * before the <ent> tag (which precedes the <hw>).
466 *
467 * Re-create the first <p>...</p> block from the data collected so
468 * far and push it as the first element to the newly created top-
469 * level tag list. Set first_p_seen flag to make sure this code is
470 * executed once.
471 *
472 * See also the comment to first_p_seen, above.
473 */
474 struct gcide_tag *tag = current_tag;
475 tag->tag_type = gcide_content_tag;
476 tag->v.tag.tag_parmc = 1;
477 tag->v.tag.tag_parmv = calloc(2, sizeof(tag->v.tag.tag_parmv[0]));
478 if (!tag->v.tag.tag_parmv)
479 memerr("pop_tag");
480 tag->v.tag.tag_parmv[0] = strdup("p");
481 if (!tag->v.tag.tag_parmv[0])
482 memerr("pop_tag");
483 tag->v.tag.tag_parmv[1] = NULL;
484 current_tag = alloc_tag(gcide_content_top);
485 append_tag(tag);
486 first_p_seen = 1;
487 } else {
488 size_t i, count;
489
490 /* Try to find matching entry */
491 count = dico_list_count(tagstk);
492 for (i = count; i > 0; i--) {
493 struct gcide_tag *p = dico_list_item(tagstk, i-1);
494 if (p->tag_type == gcide_content_tag &&
495 strlen(p->tag_name) == len &&
496 memcmp(p->tag_name, tagstr, len) == 0) {
497 do {
498 if (current_tag->tag_type == gcide_content_tag)
499 dico_log(L_WARN, 0, "%s:%zu: %s: unclosed tag",
500 LOCUS_FILE, LOCUS_OFF(current_tag->offset),
501 current_tag->tag_name);
502 } while ((current_tag = dico_list_pop(tagstk)) != p);
503 current_tag = dico_list_pop(tagstk);
504 return;
505 }
506 }
507 dico_log(L_WARN, 0, "%s:%zu: %*.*s: unexpected closing tag",
508 LOCUS_FILE, LOCUS_OFF(token_beg), len, len, tagstr);
509 }
510 }
511
512 struct walk_closure {
513 int (*fun)(int, struct gcide_tag *, void *);
514 void *data;
515 };
516
517 static int
518 inorder_helper(void *item, void *data)
519 {
520 struct gcide_tag *tag = item;
521 struct walk_closure *cp = data;
522
523 if (cp->fun(0, tag, cp->data))
524 return 1;
525 if (tag->tag_type == gcide_content_top ||
526 tag->tag_type == gcide_content_tag) {
527 dico_list_iterate(tag->v.tag.taglist, inorder_helper, data);
528 if (cp->fun(1, tag, cp->data))
529 return 1;
530 }
531 return 0;
532 }
533
534 int
535 gcide_parse_tree_inorder(struct gcide_parse_tree *tp,
536 int (*fun)(int, struct gcide_tag *, void *),
537 void *data)
538 {
539 struct walk_closure clos;
540 clos.fun = fun;
541 clos.data = data;
542 return inorder_helper(tp->root, &clos);
543 }
544
545 static int
546 tag_fixup(void *item, void *data)
547 {
548 struct gcide_tag *tag = item;
549 char *textspace = data;
550
551 switch (tag->tag_type) {
552 case gcide_content_text:
553 tag->v.text = textspace + tag->v.textpos;
554 break;
555 case gcide_content_top:
556 case gcide_content_tag:
557 dico_list_iterate(tag->v.tag.taglist, tag_fixup, textspace);
558 break;
559 default:
560 break;
561 }
562 return 0;
563 }
564
565 static char const *block_tags[] = {
566 "source",
567 "sn",
568 "p",
569 "q",
570 "rj",
571 "gcide_quote",
572 NULL
573 };
574
575 int
576 gcide_is_block_tag(struct gcide_tag *tag)
577 {
578 if (tag && tag->tag_type == gcide_content_tag) {
579 char const **p;
580
581 for (p = block_tags; *p; p++)
582 if (strcmp(tag->tag_name, *p) == 0)
583 return 1;
584 }
585 return 0;
586 }
587
588 int
589 gcide_is_ws_text(struct gcide_tag *tag)
590 {
591 if (tag && tag->tag_type == gcide_content_text) {
592 char *p;
593 for (p = tag->v.text; *p == '\n' || *p == ' ' || *p == '\t'; p++);
594 return *p == 0;
595 }
596 return 0;
597 }
598
599 /*
600 * Fix up the nl/br usage. See the comment to ws_fixup, below.
601 */
602 static int
603 ws_reduce(void *a, void *b, void *data)
604 {
605 struct gcide_tag *atag = a;
606 struct gcide_tag *btag = b;
607
608 if (atag->delete)
609 return 1;
610
611 switch (atag->tag_type) {
612 case gcide_content_tag:
613 if (strcmp(atag->tag_name, "source") == 0 &&
614 btag &&
615 btag->tag_type == gcide_content_text &&
616 btag->v.text[0] == ']')
617 btag->v.text++;
618 dico_list_reduce(atag->v.tag.taglist, ws_reduce, NULL);
619 if (gcide_is_block_tag(atag) && gcide_is_ws_text(btag))
620 btag->delete = 1;
621 break;
622
623 case gcide_content_text:
624 if (atag->v.text[0] == 0)
625 return 1;
626 else if (gcide_is_ws_text(atag) && gcide_is_block_tag(btag))
627 return 1;
628 else {
629 size_t len = strlen(atag->v.text);
630 if (atag->v.text[len-1] == '[' &&
631 btag &&
632 btag->tag_type == gcide_content_tag &&
633 strcmp(btag->tag_name, "source") == 0) {
634 if (len == 1)
635 return 1;
636 atag->v.text[len-1] = 0;
637 }
638 }
639 break;
640
641 case gcide_content_nl:
642 if (btag == NULL)
643 return 1;
644 switch (btag->tag_type) {
645 case gcide_content_tag:
646 return gcide_is_block_tag(btag);
647
648 case gcide_content_text:
649 if (btag->v.text[0] == ' ' || btag->v.text[0] == '\t')
650 return 1;
651 break;
652
653 case gcide_content_nl:
654 case gcide_content_br:
655 return 1;
656
657 default:
658 break;
659 }
660 break;
661
662 case gcide_content_br:
663 if (btag != NULL) {
664 if (btag->tag_type == gcide_content_nl)
665 btag->delete = 1;
666 }
667 break;
668
669 default:
670 break;
671 }
672 return 0;
673 }
674
675 /*
676 * This analyzes the placement of gcide_content_nl and gcide_content_br
677 * elements and removes the former where they are superfluous.
678 * The gcide_content_nl elements are removed when they appear at either
679 * side of a gcide_content_br element, or before a block tag, or a text
680 * segment that begins with a whitespace character.
681 *
682 * Additionally, curly brackets surrounding a <source>...</source> construct
683 * are removed as well.
684 */
685 static void
686 ws_fixup(struct gcide_tag *tag)
687 {
688 if (tag->tag_type == gcide_content_top ||
689 tag->tag_type == gcide_content_tag)
690 dico_list_reduce(tag->v.tag.taglist, ws_reduce, NULL);
691 }
692
693 static size_t
694 greek_translit(size_t n)
695 {
696 while (textspace[n]) {
697 size_t rd;
698 const char *greek = gcide_grk_to_utf8(textspace + n, &rd);
699
700 if (greek) {
701 text_add_str(greek, strlen(greek));
702 n += rd;
703 } else {
704 text_add_chr(textspace[n]);
705 n++;
706 }
707 }
708 return text_segment_finish();
709 }
710
711 static int
712 early_fixup(void *item, void *data)
713 {
714 struct gcide_tag *tag = item;
715 int translate = data ? *(int*)data : 0;
716
717 switch (tag->tag_type) {
718 case gcide_content_text:
719 if (translate)
720 tag->v.textpos = greek_translit(tag->v.textpos);
721 break;
722 case gcide_content_top:
723 dico_list_iterate(tag->v.tag.taglist, early_fixup, &translate);
724 break;
725 case gcide_content_tag:
726 translate = gcide_is_tag(tag, "grk");
727 dico_list_iterate(tag->v.tag.taglist, early_fixup, &translate);
728 break;
729 default:
730 break;
731 }
732 return 0;
733 }
734
735 /*
736 * The <as> blocks in GCIDE corpus often begin with "as, ". This looks strange
737 * if another font is used for <as> content or if it is otherwise marked up.
738 * To avoid this, move "as, " out of <as>block.
739 */
740 static void
741 as_fixup(struct gcide_tag *tag)
742 {
743 dico_iterator_t itr;
744
745 switch (tag->tag_type) {
746 case gcide_content_top:
747 case gcide_content_tag:
748 itr = dico_list_iterator(tag->v.tag.taglist);
749 tag = dico_iterator_first(itr);
750 while (tag) {
751 if (gcide_is_tag(tag, "as")) {
752 struct gcide_tag *t = dico_list_head(tag->v.tag.taglist);
753 char *s = textspace + t->v.textpos;
754 if (strncmp(s, "as", 2) == 0 &&
755 (isspace(s[3]) || ispunct(s[3]))) {
756 int len;
757 struct gcide_tag *t1;
758
759 for (len = 3; s[len] && isspace(s[len]); len++);
760 text_add_str(s, len);
761 t1 = alloc_tag(gcide_content_text);
762 t1->v.textpos = text_segment_finish();
763 dico_iterator_insert_before(itr, t1);
764 dico_list_remove(tag->v.tag.taglist, t, NULL);
765 if (s[len]) {
766 t1 = alloc_tag(gcide_content_text);
767 text_add_strz(s + len);
768 t1->v.textpos = text_segment_finish();
769 dico_list_prepend(tag->v.tag.taglist, t1);
770 }
771 }
772 } else
773 as_fixup(tag);
774 tag = dico_iterator_next(itr);
775 }
776 dico_iterator_destroy(&itr);
777 break;
778 default:
779 break;
780 }
781 }
782
783 struct gcide_parse_tree *
784 gcide_markup_parse(char const *text, size_t len, int dbg, struct gcide_locus *loc)
785 {
786 struct gcide_parse_tree *tp;
787 struct gcide_tag *p;
788
789 input_buf = text;
790 input_len = len;
791 token_beg = token_end = 0;
792 first_p_seen = 0;
793 base_locus = loc;
794
795 if (setjmp(errbuf)) {
796 free(textspace);
797 dico_list_destroy(&tagstk);
798 free_tag(current_tag, NULL);
799 }
800
801 if (SIZE_MAX / 2 < len)
802 memerr("gcide_markup_parse");
803 textsize = 2 * len;
804 textspace = malloc(textsize);
805 if (!textspace)
806 memerr("gcide_markup_parse");
807 textpos = textstart = 0;
808
809 tagstk = dico_list_create();
810 if (!tagstk)
811 memerr("gcide_markup_parse");
812 dico_list_set_free_item(tagstk, free_tag, NULL);
813 current_tag = alloc_tag(gcide_content_top);
814
815 yy_flex_debug = dbg;
816 BEGIN(INITIAL);
817 while (yylex())
818 ;
819
820 /* Append trailing text segment, if any */
821 flush_text_segment();
822
823 if (current_tag->tag_type != gcide_content_top) {
824 p = current_tag;
825 do {
826 if (p->tag_type == gcide_content_tag)
827 dico_log(L_WARN, 0, "%s:%zu: unclosed %s tag",
828 LOCUS_FILE, LOCUS_OFF(p->offset), p->tag_name);
829 current_tag = p;
830 } while ((p = dico_list_pop(tagstk)));
831 }
832
833 dico_list_destroy(&tagstk);
834
835 if (!current_tag) {
836 free(textspace);
837 return NULL;
838 }
839
840 early_fixup(current_tag, NULL);
841 as_fixup(current_tag);
842
843 tp = malloc(sizeof(*tp));
844 if (!tp)
845 memerr("gcide_markup_parse");
846
847 tp->textspace = textspace;
848 tp->textsize = textsize;
849
850 tp->root = current_tag;
851
852 tag_fixup(tp->root, textspace);
853 ws_fixup(tp->root);
854
855 return tp;
856 }
857
858 void
859 gcide_parse_tree_free(struct gcide_parse_tree *tp)
860 {
861 free(tp->textspace);
862 free_tag(tp->root, NULL);
863 free(tp);
864 }