%top { /* This file is part of GNU Dico. Copyright (C) 2012-2024 Sergey Poznyakoff GNU Dico is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3, or (at your option) any later version. GNU Dico is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with GNU Dico. If not, see . */ #include #include #include #include #include #include #include #include #include #include #include #include "gcide.h" #include "wordsplit.h" #define yy_create_buffer gcide_markup_yy_create_buffer #define yy_delete_buffer gcide_markup_yy_delete_buffer #define yy_flex_debug gcide_markup_yy_flex_debug #define yy_init_buffer gcide_markup_yy_init_buffer #define yy_flush_buffer gcide_markup_yy_flush_buffer #define yy_load_buffer_state gcide_markup_yy_load_buffer_state #define yy_switch_to_buffer gcide_markup_yy_switch_to_buffer #define yyin gcide_markup_yyin #define yyleng gcide_markup_yyleng #define yylex gcide_markup_yylex #define yylineno gcide_markup_yylineno #define yyout gcide_markup_yyout #define yyrestart gcide_markup_yyrestart #define yytext gcide_markup_yytext #define yywrap gcide_markup_yywrap #define yyalloc gcide_markup_yyalloc #define yyrealloc gcide_markup_yyrealloc #define yyfree gcide_markup_yyfree #define yyunput gcide_markup_yyunput #define yylex_destroy gcide_markup_yylex_destroy #define yyget_debug gcide_markup_yyget_debug #define yyset_debug gcide_markup_yyset_debug #define yyget_extra gcide_markup_yyget_extra #define yyset_extra gcide_markup_yyset_extra #define yyget_in gcide_markup_yyget_in #define yyset_in gcide_markup_yyset_in #define yyget_out gcide_markup_yyget_out #define yyset_out gcide_markup_yyset_out #define yyget_leng gcide_markup_yyget_leng #define yyget_text gcide_markup_yyget_text #define yyget_lineno gcide_markup_yyget_lineno #define yyset_lineno gcide_markup_yyset_lineno } %{ static struct gcide_locus *base_locus; static char const *input_buf; static size_t input_len; static unsigned token_beg; static unsigned token_end; /* * Dictionary entries in GCIDE corpus files begin with "

", but index * entries in the GCIDE.IDX file point to the first "" tag that follows. * Thus, the very first "

" encountered when parsing an entry has no * corresponding "

" tag. The first_p_seen variable is set by push_tag * when it sees the first opening "

" tag and is used in pop_tag to generate * the first "p" block tag and avoid issuing a spurious error message when * the first "

" is encountered. */ static int first_p_seen; #define LOCUS_FILE (base_locus ? base_locus->file : "") #define LOCUS_OFF(off) ((base_locus ? base_locus->offset : 0) + off) static char *textspace; /* Text storage space */ static size_t textsize; /* Size of text space */ static size_t textpos; /* Current position in the text space */ static size_t textstart; /* Start of the current text segment */ static size_t textstart_offset; static dico_list_t tagstk; static struct gcide_tag *current_tag; static jmp_buf errbuf; static void memerr(const char *text) { dico_log(L_ERR, ENOMEM, "%s", text); longjmp(errbuf, 1); } #define YY_USER_ACTION do { \ token_beg = token_end; \ token_end += yyleng; \ } while (0); #define YY_INPUT(buf,result,max_size) do { \ size_t __n = (max_size) > input_len ? input_len : (max_size); \ if (__n) \ memcpy((buf), input_buf, __n); \ input_buf += __n; \ input_len -= __n; \ (result) = __n; \ } while(0) static int retstate; static char *endtag; static size_t endtaglen; #define BEGIN_COMMENT(end) \ { retstate = YYSTATE; endtag = (end); endtaglen = strlen(endtag); BEGIN(COMMENT); } static void text_assert_size(size_t len) { while (textsize - textpos < len) { size_t nsize; char *newp; if (SIZE_MAX / 2 < textsize) memerr("text_add_str"); nsize = 2 * textsize; newp = realloc(textspace, nsize); if (!newp) memerr("text_add_str"); textspace = newp; textsize = nsize; } } static void text_add_str(char const *s, size_t l) { if (textstart == textpos) textstart_offset = token_beg; text_assert_size(l); memcpy(textspace + textpos, s, l); textpos += l; } static void text_add_strz(char const *s) { text_add_str(s, strlen(s)); } static void text_add_chr(int ch) { char c = ch; text_add_str(&c, 1); } #define text_segment_length() (textpos - textstart) static size_t text_segment_finish() { size_t ret = textstart; text_add_chr(0); textstart = textpos; return ret; } static int free_tag(void *item, void *data); static struct gcide_tag * alloc_tag(enum gcide_content_type type) { dico_list_t list; struct gcide_tag *tag = calloc(1, sizeof(*tag)); if (!tag) memerr("alloc_tag"); tag->tag_type = type; switch (type) { case gcide_content_top: case gcide_content_tag: list = dico_list_create(); if (!list) { free(tag); memerr("alloc_tag"); } dico_list_set_free_item(list, free_tag, NULL); tag->v.tag.taglist = list; break; default: break; } return tag; } struct gcide_tag * gcide_tag_alloc(const char *text, size_t len) { struct gcide_tag *tag; struct wordsplit ws; size_t i; if (wordsplit_len(text, len, &ws, WRDSF_DEFFLAGS & ~WRDSF_CESCAPES)) { dico_log(L_ERR, 0, _("cannot parse line %.*s: %s"), (int)len, text, wordsplit_strerror(&ws)); longjmp(errbuf, 1); } for (i = 0; i < ws.ws_wordc; i++) { char *p; for (p = ws.ws_wordv[i]; *p; p++) { if (*p == '=') break; *p = tolower(*p); } } tag = alloc_tag(gcide_content_tag); tag->v.tag.tag_parmc = ws.ws_wordc; tag->v.tag.tag_parmv = ws.ws_wordv; tag->offset = token_beg; ws.ws_wordc = 0; ws.ws_wordv = NULL; wordsplit_free(&ws); return tag; } #define alloc_taglist gcide_tag_alloc void gcide_tag_free(struct gcide_tag *tag) { if (tag) { switch (tag->tag_type) { default: break; case gcide_content_top: case gcide_content_tag: dico_list_destroy(&tag->v.tag.taglist); } free(tag); } } int gcide_is_tag(struct gcide_tag *tag, char const *name) { return tag != NULL && tag->tag_type == gcide_content_tag && strcmp(tag->tag_name, name) == 0; } static int free_tag(void *item, void *data) { gcide_tag_free(item); return 0; } static void append_tag(struct gcide_tag *tag) { dico_list_t list; switch (current_tag->tag_type) { case gcide_content_text: { struct gcide_tag *subtag = alloc_tag(gcide_content_text); subtag->offset = current_tag->offset; subtag->v.textpos = current_tag->v.textpos; list = dico_list_create(); if (!list) { free(subtag); free(tag); memerr("append_tag"); } dico_list_set_free_item(list, free_tag, NULL); dico_list_append(list, subtag); current_tag->tag_type = gcide_content_tag; current_tag->v.tag.taglist = list; break; } default: break; } dico_list_append(current_tag->v.tag.taglist, tag); } static void push_tag(struct gcide_tag *tag); static void pop_tag(const char *tagstr, size_t taglen); static void flush_text_segment(void) { if (text_segment_length()) { struct gcide_tag *tag = alloc_tag(gcide_content_text); tag->v.textpos = text_segment_finish(); tag->offset = textstart_offset; append_tag(tag); } } %} %option 8bit %option nounput %option noinput %option nodefault %x COMMENT PR HW PRE XD [0-9a-f] WS [ \t] %% { "<--" BEGIN_COMMENT("-->"); ""); "<"[a-zA-Z][a-zA-Z0-9._-]*">" { flush_text_segment(); push_tag(alloc_taglist(yytext + 1, yyleng - 2)); } "<"[a-zA-Z][a-zA-Z0-9._-]*{WS}[^>]*">" { flush_text_segment(); push_tag(alloc_taglist(yytext + 1, yyleng - 2)); } "" { flush_text_segment(); pop_tag(yytext + 2, yyleng - 3); } "]*"/" { dico_log(L_WARN, 0, _("%s:%zu: malformed entity or tag: %s"), LOCUS_FILE, LOCUS_OFF(token_beg), yytext); } "
{ \n+ { flush_text_segment(); append_tag(alloc_tag(gcide_content_nl)); } } { [^<\\\r\n]+ text_add_str(yytext, yyleng); }
{
  [^<\\\r]+ text_add_str(yytext, yyleng);
}
{
  [""`*]       ;
  [^<\\\r""`*]+ text_add_str(yytext, yyleng);
}
{
  "\""         text_add_strz("ˈ"); /* Main accent */
  "`"          text_add_strz("ˌ"); /* Secondary accent */
  "*"          text_add_strz("‑"); /* Syllable break */
  [^<\\\r""`*]+  text_add_str(yytext, yyleng);
}
{
  [^>]+">" {
      if (yyleng >= endtaglen &&
	  memcmp(yytext + yyleng - endtaglen, endtag, endtaglen) == 0)
	  BEGIN(retstate);
  }
  .|\n         ;
}

.  {
	if (input_len > 0)
		dico_log(L_NOTICE, 0,
			 _("%s:%zu: unmatched character %03o; please report"),
			 LOCUS_FILE, LOCUS_OFF(token_beg), yytext[0]);
	text_add_chr(yytext[0]);
}
%%

int
yywrap()
{
    return 1;
}

static void
push_tag(struct gcide_tag *tag)
{
    append_tag(tag);
    dico_list_push(tagstk, current_tag);
    current_tag = tag;
    if (strcmp(current_tag->tag_name, "pr") == 0)
	    BEGIN(PR);
    else if (strcmp(current_tag->tag_name, "hw") == 0)
	    BEGIN(HW);
    else if (strcmp(current_tag->tag_name, "pre") == 0)
	    BEGIN(PRE);
    else if (strcmp(current_tag->tag_name, "p") == 0)
	    first_p_seen = 1;
}

static void
pop_tag(const char *tagstr, size_t taglen)
{
    int len;
    char *p;

    text_assert_size(taglen);
    p = textspace + textpos;
    for (len = 0; len < taglen; len++) {
	if (tagstr[len] == ' ' || tagstr[len] == '\t')
	    break;
	*p++ = tolower(tagstr[len]);
    }
    tagstr = textspace + textpos;

    if ((len == 2 && (memcmp(tagstr, "pr", len) == 0 ||
		      memcmp(tagstr, "hw", len) == 0)) ||
	(len == 3 && memcmp(tagstr, "pre", len) == 0))
	BEGIN(INITIAL);

    if (current_tag->tag_type == gcide_content_tag &&
	strlen(current_tag->tag_name) == len &&
	memcmp(current_tag->tag_name, tagstr, len) == 0)
	current_tag = dico_list_pop(tagstk);
    else if (current_tag->tag_type == gcide_content_top &&
	     len == 1 &&
	     tagstr[0] == 'p' &&
	     !first_p_seen) {
	/*
	 * The first instance of 

has no corresponding open

tag * because idxgcide stores the offset of the first tag in the * index entry, whereas the opening

tag in GCIDE entries appears * before the tag (which precedes the ). * * Re-create the first

...

block from the data collected so * far and push it as the first element to the newly created top- * level tag list. Set first_p_seen flag to make sure this code is * executed once. * * See also the comment to first_p_seen, above. */ struct gcide_tag *tag = current_tag; tag->tag_type = gcide_content_tag; tag->v.tag.tag_parmc = 1; tag->v.tag.tag_parmv = calloc(2, sizeof(tag->v.tag.tag_parmv[0])); if (!tag->v.tag.tag_parmv) memerr("pop_tag"); tag->v.tag.tag_parmv[0] = strdup("p"); if (!tag->v.tag.tag_parmv[0]) memerr("pop_tag"); tag->v.tag.tag_parmv[1] = NULL; current_tag = alloc_tag(gcide_content_top); append_tag(tag); first_p_seen = 1; } else { size_t i, count; /* Try to find matching entry */ count = dico_list_count(tagstk); for (i = count; i > 0; i--) { struct gcide_tag *p = dico_list_item(tagstk, i-1); if (p->tag_type == gcide_content_tag && strlen(p->tag_name) == len && memcmp(p->tag_name, tagstr, len) == 0) { do { if (current_tag->tag_type == gcide_content_tag) dico_log(L_WARN, 0, "%s:%zu: %s: unclosed tag", LOCUS_FILE, LOCUS_OFF(current_tag->offset), current_tag->tag_name); } while ((current_tag = dico_list_pop(tagstk)) != p); current_tag = dico_list_pop(tagstk); return; } } dico_log(L_WARN, 0, "%s:%zu: %*.*s: unexpected closing tag", LOCUS_FILE, LOCUS_OFF(token_beg), len, len, tagstr); } } struct walk_closure { int (*fun)(int, struct gcide_tag *, void *); void *data; }; static int inorder_helper(void *item, void *data) { struct gcide_tag *tag = item; struct walk_closure *cp = data; if (cp->fun(0, tag, cp->data)) return 1; if (tag->tag_type == gcide_content_top || tag->tag_type == gcide_content_tag) { dico_list_iterate(tag->v.tag.taglist, inorder_helper, data); if (cp->fun(1, tag, cp->data)) return 1; } return 0; } int gcide_parse_tree_inorder(struct gcide_parse_tree *tp, int (*fun)(int, struct gcide_tag *, void *), void *data) { struct walk_closure clos; clos.fun = fun; clos.data = data; return inorder_helper(tp->root, &clos); } static int tag_fixup(void *item, void *data) { struct gcide_tag *tag = item; char *textspace = data; switch (tag->tag_type) { case gcide_content_text: tag->v.text = textspace + tag->v.textpos; break; case gcide_content_top: case gcide_content_tag: dico_list_iterate(tag->v.tag.taglist, tag_fixup, textspace); break; default: break; } return 0; } static char const *block_tags[] = { "source", "sn", "p", "q", "rj", "gcide_quote", NULL }; int gcide_is_block_tag(struct gcide_tag *tag) { if (tag && tag->tag_type == gcide_content_tag) { char const **p; for (p = block_tags; *p; p++) if (strcmp(tag->tag_name, *p) == 0) return 1; } return 0; } int gcide_is_ws_text(struct gcide_tag *tag) { if (tag && tag->tag_type == gcide_content_text) { char *p; for (p = tag->v.text; *p == '\n' || *p == ' ' || *p == '\t'; p++); return *p == 0; } return 0; } /* * Fix up the nl/br usage. See the comment to ws_fixup, below. */ static int ws_reduce(void *a, void *b, void *data) { struct gcide_tag *atag = a; struct gcide_tag *btag = b; if (atag->delete) return 1; switch (atag->tag_type) { case gcide_content_tag: if (strcmp(atag->tag_name, "source") == 0 && btag && btag->tag_type == gcide_content_text && btag->v.text[0] == ']') btag->v.text++; dico_list_reduce(atag->v.tag.taglist, ws_reduce, NULL); if (gcide_is_block_tag(atag) && gcide_is_ws_text(btag)) btag->delete = 1; break; case gcide_content_text: if (atag->v.text[0] == 0) return 1; else if (gcide_is_ws_text(atag) && gcide_is_block_tag(btag)) return 1; else { size_t len = strlen(atag->v.text); if (atag->v.text[len-1] == '[' && btag && btag->tag_type == gcide_content_tag && strcmp(btag->tag_name, "source") == 0) { if (len == 1) return 1; atag->v.text[len-1] = 0; } } break; case gcide_content_nl: if (btag == NULL) return 1; switch (btag->tag_type) { case gcide_content_tag: return gcide_is_block_tag(btag); case gcide_content_text: if (btag->v.text[0] == ' ' || btag->v.text[0] == '\t') return 1; break; case gcide_content_nl: case gcide_content_br: return 1; default: break; } break; case gcide_content_br: if (btag != NULL) { if (btag->tag_type == gcide_content_nl) btag->delete = 1; } break; default: break; } return 0; } /* * This analyzes the placement of gcide_content_nl and gcide_content_br * elements and removes the former where they are superfluous. * The gcide_content_nl elements are removed when they appear at either * side of a gcide_content_br element, or before a block tag, or a text * segment that begins with a whitespace character. * * Additionally, curly brackets surrounding a ... construct * are removed as well. */ static void ws_fixup(struct gcide_tag *tag) { if (tag->tag_type == gcide_content_top || tag->tag_type == gcide_content_tag) dico_list_reduce(tag->v.tag.taglist, ws_reduce, NULL); } static size_t greek_translit(size_t n) { while (textspace[n]) { size_t rd; const char *greek = gcide_grk_to_utf8(textspace + n, &rd); if (greek) { text_add_str(greek, strlen(greek)); n += rd; } else { text_add_chr(textspace[n]); n++; } } return text_segment_finish(); } static int early_fixup(void *item, void *data) { struct gcide_tag *tag = item; int translate = data ? *(int*)data : 0; switch (tag->tag_type) { case gcide_content_text: if (translate) tag->v.textpos = greek_translit(tag->v.textpos); break; case gcide_content_top: dico_list_iterate(tag->v.tag.taglist, early_fixup, &translate); break; case gcide_content_tag: translate = gcide_is_tag(tag, "grk"); dico_list_iterate(tag->v.tag.taglist, early_fixup, &translate); break; default: break; } return 0; } /* * The blocks in GCIDE corpus often begin with "as, ". This looks strange * if another font is used for content or if it is otherwise marked up. * To avoid this, move "as, " out of block. */ static void as_fixup(struct gcide_tag *tag) { dico_iterator_t itr; switch (tag->tag_type) { case gcide_content_top: case gcide_content_tag: itr = dico_list_iterator(tag->v.tag.taglist); tag = dico_iterator_first(itr); while (tag) { if (gcide_is_tag(tag, "as")) { struct gcide_tag *t = dico_list_head(tag->v.tag.taglist); char *s = textspace + t->v.textpos; if (strncmp(s, "as", 2) == 0 && (isspace(s[3]) || ispunct(s[3]))) { int len; struct gcide_tag *t1; for (len = 3; s[len] && isspace(s[len]); len++); text_add_str(s, len); t1 = alloc_tag(gcide_content_text); t1->v.textpos = text_segment_finish(); dico_iterator_insert_before(itr, t1); dico_list_remove(tag->v.tag.taglist, t, NULL); if (s[len]) { t1 = alloc_tag(gcide_content_text); text_add_strz(s + len); t1->v.textpos = text_segment_finish(); dico_list_prepend(tag->v.tag.taglist, t1); } } } else as_fixup(tag); tag = dico_iterator_next(itr); } dico_iterator_destroy(&itr); break; default: break; } } struct gcide_parse_tree * gcide_markup_parse(char const *text, size_t len, int dbg, struct gcide_locus *loc) { struct gcide_parse_tree *tp; struct gcide_tag *p; input_buf = text; input_len = len; token_beg = token_end = 0; first_p_seen = 0; base_locus = loc; if (setjmp(errbuf)) { free(textspace); dico_list_destroy(&tagstk); free_tag(current_tag, NULL); } if (SIZE_MAX / 2 < len) memerr("gcide_markup_parse"); textsize = 2 * len; textspace = malloc(textsize); if (!textspace) memerr("gcide_markup_parse"); textpos = textstart = 0; tagstk = dico_list_create(); if (!tagstk) memerr("gcide_markup_parse"); dico_list_set_free_item(tagstk, free_tag, NULL); current_tag = alloc_tag(gcide_content_top); yy_flex_debug = dbg; BEGIN(INITIAL); while (yylex()) ; /* Append trailing text segment, if any */ flush_text_segment(); if (current_tag->tag_type != gcide_content_top) { p = current_tag; do { if (p->tag_type == gcide_content_tag) dico_log(L_WARN, 0, "%s:%zu: unclosed %s tag", LOCUS_FILE, LOCUS_OFF(p->offset), p->tag_name); current_tag = p; } while ((p = dico_list_pop(tagstk))); } dico_list_destroy(&tagstk); if (!current_tag) { free(textspace); return NULL; } early_fixup(current_tag, NULL); as_fixup(current_tag); tp = malloc(sizeof(*tp)); if (!tp) memerr("gcide_markup_parse"); tp->textspace = textspace; tp->textsize = textsize; tp->root = current_tag; tag_fixup(tp->root, textspace); ws_fixup(tp->root); return tp; } void gcide_parse_tree_free(struct gcide_parse_tree *tp) { free(tp->textspace); free_tag(tp->root, NULL); free(tp); }