%top {
/* This file is part of GNU Dico.
Copyright (C) 2012-2024 Sergey Poznyakoff
GNU Dico is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3, or (at your option)
any later version.
GNU Dico is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with GNU Dico. If not, see . */
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include "gcide.h"
#include "wordsplit.h"
#define yy_create_buffer gcide_markup_yy_create_buffer
#define yy_delete_buffer gcide_markup_yy_delete_buffer
#define yy_flex_debug gcide_markup_yy_flex_debug
#define yy_init_buffer gcide_markup_yy_init_buffer
#define yy_flush_buffer gcide_markup_yy_flush_buffer
#define yy_load_buffer_state gcide_markup_yy_load_buffer_state
#define yy_switch_to_buffer gcide_markup_yy_switch_to_buffer
#define yyin gcide_markup_yyin
#define yyleng gcide_markup_yyleng
#define yylex gcide_markup_yylex
#define yylineno gcide_markup_yylineno
#define yyout gcide_markup_yyout
#define yyrestart gcide_markup_yyrestart
#define yytext gcide_markup_yytext
#define yywrap gcide_markup_yywrap
#define yyalloc gcide_markup_yyalloc
#define yyrealloc gcide_markup_yyrealloc
#define yyfree gcide_markup_yyfree
#define yyunput gcide_markup_yyunput
#define yylex_destroy gcide_markup_yylex_destroy
#define yyget_debug gcide_markup_yyget_debug
#define yyset_debug gcide_markup_yyset_debug
#define yyget_extra gcide_markup_yyget_extra
#define yyset_extra gcide_markup_yyset_extra
#define yyget_in gcide_markup_yyget_in
#define yyset_in gcide_markup_yyset_in
#define yyget_out gcide_markup_yyget_out
#define yyset_out gcide_markup_yyset_out
#define yyget_leng gcide_markup_yyget_leng
#define yyget_text gcide_markup_yyget_text
#define yyget_lineno gcide_markup_yyget_lineno
#define yyset_lineno gcide_markup_yyset_lineno
}
%{
static struct gcide_locus *base_locus;
static char const *input_buf;
static size_t input_len;
static unsigned token_beg;
static unsigned token_end;
/*
* Dictionary entries in GCIDE corpus files begin with "", but index
* entries in the GCIDE.IDX file point to the first "" tag that follows.
* Thus, the very first "
" encountered when parsing an entry has no
* corresponding "" tag. The first_p_seen variable is set by push_tag
* when it sees the first opening "
" tag and is used in pop_tag to generate
* the first "p" block tag and avoid issuing a spurious error message when
* the first "
" is encountered.
*/
static int first_p_seen;
#define LOCUS_FILE (base_locus ? base_locus->file : "")
#define LOCUS_OFF(off) ((base_locus ? base_locus->offset : 0) + off)
static char *textspace; /* Text storage space */
static size_t textsize; /* Size of text space */
static size_t textpos; /* Current position in the text space */
static size_t textstart; /* Start of the current text segment */
static size_t textstart_offset;
static dico_list_t tagstk;
static struct gcide_tag *current_tag;
static jmp_buf errbuf;
static void
memerr(const char *text)
{
dico_log(L_ERR, ENOMEM, "%s", text);
longjmp(errbuf, 1);
}
#define YY_USER_ACTION do { \
token_beg = token_end; \
token_end += yyleng; \
} while (0);
#define YY_INPUT(buf,result,max_size) do { \
size_t __n = (max_size) > input_len ? input_len : (max_size); \
if (__n) \
memcpy((buf), input_buf, __n); \
input_buf += __n; \
input_len -= __n; \
(result) = __n; \
} while(0)
static int retstate;
static char *endtag;
static size_t endtaglen;
#define BEGIN_COMMENT(end) \
{ retstate = YYSTATE; endtag = (end); endtaglen = strlen(endtag); BEGIN(COMMENT); }
static void
text_assert_size(size_t len)
{
while (textsize - textpos < len) {
size_t nsize;
char *newp;
if (SIZE_MAX / 2 < textsize)
memerr("text_add_str");
nsize = 2 * textsize;
newp = realloc(textspace, nsize);
if (!newp)
memerr("text_add_str");
textspace = newp;
textsize = nsize;
}
}
static void
text_add_str(char const *s, size_t l)
{
if (textstart == textpos) textstart_offset = token_beg;
text_assert_size(l);
memcpy(textspace + textpos, s, l);
textpos += l;
}
static void
text_add_strz(char const *s)
{
text_add_str(s, strlen(s));
}
static void
text_add_chr(int ch)
{
char c = ch;
text_add_str(&c, 1);
}
#define text_segment_length() (textpos - textstart)
static size_t
text_segment_finish()
{
size_t ret = textstart;
text_add_chr(0);
textstart = textpos;
return ret;
}
static int free_tag(void *item, void *data);
static struct gcide_tag *
alloc_tag(enum gcide_content_type type)
{
dico_list_t list;
struct gcide_tag *tag = calloc(1, sizeof(*tag));
if (!tag)
memerr("alloc_tag");
tag->tag_type = type;
switch (type) {
case gcide_content_top:
case gcide_content_tag:
list = dico_list_create();
if (!list) {
free(tag);
memerr("alloc_tag");
}
dico_list_set_free_item(list, free_tag, NULL);
tag->v.tag.taglist = list;
break;
default:
break;
}
return tag;
}
struct gcide_tag *
gcide_tag_alloc(const char *text, size_t len)
{
struct gcide_tag *tag;
struct wordsplit ws;
size_t i;
if (wordsplit_len(text, len, &ws, WRDSF_DEFFLAGS & ~WRDSF_CESCAPES)) {
dico_log(L_ERR, 0, _("cannot parse line %.*s: %s"),
(int)len, text, wordsplit_strerror(&ws));
longjmp(errbuf, 1);
}
for (i = 0; i < ws.ws_wordc; i++) {
char *p;
for (p = ws.ws_wordv[i]; *p; p++) {
if (*p == '=')
break;
*p = tolower(*p);
}
}
tag = alloc_tag(gcide_content_tag);
tag->v.tag.tag_parmc = ws.ws_wordc;
tag->v.tag.tag_parmv = ws.ws_wordv;
tag->offset = token_beg;
ws.ws_wordc = 0;
ws.ws_wordv = NULL;
wordsplit_free(&ws);
return tag;
}
#define alloc_taglist gcide_tag_alloc
void
gcide_tag_free(struct gcide_tag *tag)
{
if (tag) {
switch (tag->tag_type) {
default:
break;
case gcide_content_top:
case gcide_content_tag:
dico_list_destroy(&tag->v.tag.taglist);
}
free(tag);
}
}
int
gcide_is_tag(struct gcide_tag *tag, char const *name)
{
return tag != NULL && tag->tag_type == gcide_content_tag &&
strcmp(tag->tag_name, name) == 0;
}
static int
free_tag(void *item, void *data)
{
gcide_tag_free(item);
return 0;
}
static void
append_tag(struct gcide_tag *tag)
{
dico_list_t list;
switch (current_tag->tag_type) {
case gcide_content_text: {
struct gcide_tag *subtag = alloc_tag(gcide_content_text);
subtag->offset = current_tag->offset;
subtag->v.textpos = current_tag->v.textpos;
list = dico_list_create();
if (!list) {
free(subtag);
free(tag);
memerr("append_tag");
}
dico_list_set_free_item(list, free_tag, NULL);
dico_list_append(list, subtag);
current_tag->tag_type = gcide_content_tag;
current_tag->v.tag.taglist = list;
break;
}
default:
break;
}
dico_list_append(current_tag->v.tag.taglist, tag);
}
static void push_tag(struct gcide_tag *tag);
static void pop_tag(const char *tagstr, size_t taglen);
static void
flush_text_segment(void)
{
if (text_segment_length()) {
struct gcide_tag *tag = alloc_tag(gcide_content_text);
tag->v.textpos = text_segment_finish();
tag->offset = textstart_offset;
append_tag(tag);
}
}
%}
%option 8bit
%option nounput
%option noinput
%option nodefault
%x COMMENT PR HW PRE
XD [0-9a-f]
WS [ \t]
%%
{
"<--" BEGIN_COMMENT("-->");
"");
"<"[a-zA-Z][a-zA-Z0-9._-]*">" {
flush_text_segment();
push_tag(alloc_taglist(yytext + 1, yyleng - 2));
}
"<"[a-zA-Z][a-zA-Z0-9._-]*{WS}[^>]*">" {
flush_text_segment();
push_tag(alloc_taglist(yytext + 1, yyleng - 2));
}
""[a-zA-Z][a-zA-Z0-9._-]*">" {
flush_text_segment();
pop_tag(yytext + 2, yyleng - 3);
}
""[a-zA-Z][^>]*"/" {
dico_log(L_WARN, 0, _("%s:%zu: malformed entity or tag: %s"),
LOCUS_FILE, LOCUS_OFF(token_beg), yytext);
}
" {
\n+ {
flush_text_segment();
append_tag(alloc_tag(gcide_content_nl));
}
}
{
[^<\\\r\n]+ text_add_str(yytext, yyleng);
}
{
[^<\\\r]+ text_add_str(yytext, yyleng);
}
{
[""`*] ;
[^<\\\r""`*]+ text_add_str(yytext, yyleng);
}
{
"\"" text_add_strz("ˈ"); /* Main accent */
"`" text_add_strz("ˌ"); /* Secondary accent */
"*" text_add_strz("‑"); /* Syllable break */
[^<\\\r""`*]+ text_add_str(yytext, yyleng);
}
{
[^>]+">" {
if (yyleng >= endtaglen &&
memcmp(yytext + yyleng - endtaglen, endtag, endtaglen) == 0)
BEGIN(retstate);
}
.|\n ;
}
. {
if (input_len > 0)
dico_log(L_NOTICE, 0,
_("%s:%zu: unmatched character %03o; please report"),
LOCUS_FILE, LOCUS_OFF(token_beg), yytext[0]);
text_add_chr(yytext[0]);
}
%%
int
yywrap()
{
return 1;
}
static void
push_tag(struct gcide_tag *tag)
{
append_tag(tag);
dico_list_push(tagstk, current_tag);
current_tag = tag;
if (strcmp(current_tag->tag_name, "pr") == 0)
BEGIN(PR);
else if (strcmp(current_tag->tag_name, "hw") == 0)
BEGIN(HW);
else if (strcmp(current_tag->tag_name, "pre") == 0)
BEGIN(PRE);
else if (strcmp(current_tag->tag_name, "p") == 0)
first_p_seen = 1;
}
static void
pop_tag(const char *tagstr, size_t taglen)
{
int len;
char *p;
text_assert_size(taglen);
p = textspace + textpos;
for (len = 0; len < taglen; len++) {
if (tagstr[len] == ' ' || tagstr[len] == '\t')
break;
*p++ = tolower(tagstr[len]);
}
tagstr = textspace + textpos;
if ((len == 2 && (memcmp(tagstr, "pr", len) == 0 ||
memcmp(tagstr, "hw", len) == 0)) ||
(len == 3 && memcmp(tagstr, "pre", len) == 0))
BEGIN(INITIAL);
if (current_tag->tag_type == gcide_content_tag &&
strlen(current_tag->tag_name) == len &&
memcmp(current_tag->tag_name, tagstr, len) == 0)
current_tag = dico_list_pop(tagstk);
else if (current_tag->tag_type == gcide_content_top &&
len == 1 &&
tagstr[0] == 'p' &&
!first_p_seen) {
/*
* The first instance of
has no corresponding open tag
* because idxgcide stores the offset of the first tag in the
* index entry, whereas the opening tag in GCIDE entries appears
* before the tag (which precedes the ).
*
* Re-create the first ...
block from the data collected so
* far and push it as the first element to the newly created top-
* level tag list. Set first_p_seen flag to make sure this code is
* executed once.
*
* See also the comment to first_p_seen, above.
*/
struct gcide_tag *tag = current_tag;
tag->tag_type = gcide_content_tag;
tag->v.tag.tag_parmc = 1;
tag->v.tag.tag_parmv = calloc(2, sizeof(tag->v.tag.tag_parmv[0]));
if (!tag->v.tag.tag_parmv)
memerr("pop_tag");
tag->v.tag.tag_parmv[0] = strdup("p");
if (!tag->v.tag.tag_parmv[0])
memerr("pop_tag");
tag->v.tag.tag_parmv[1] = NULL;
current_tag = alloc_tag(gcide_content_top);
append_tag(tag);
first_p_seen = 1;
} else {
size_t i, count;
/* Try to find matching entry */
count = dico_list_count(tagstk);
for (i = count; i > 0; i--) {
struct gcide_tag *p = dico_list_item(tagstk, i-1);
if (p->tag_type == gcide_content_tag &&
strlen(p->tag_name) == len &&
memcmp(p->tag_name, tagstr, len) == 0) {
do {
if (current_tag->tag_type == gcide_content_tag)
dico_log(L_WARN, 0, "%s:%zu: %s: unclosed tag",
LOCUS_FILE, LOCUS_OFF(current_tag->offset),
current_tag->tag_name);
} while ((current_tag = dico_list_pop(tagstk)) != p);
current_tag = dico_list_pop(tagstk);
return;
}
}
dico_log(L_WARN, 0, "%s:%zu: %*.*s: unexpected closing tag",
LOCUS_FILE, LOCUS_OFF(token_beg), len, len, tagstr);
}
}
struct walk_closure {
int (*fun)(int, struct gcide_tag *, void *);
void *data;
};
static int
inorder_helper(void *item, void *data)
{
struct gcide_tag *tag = item;
struct walk_closure *cp = data;
if (cp->fun(0, tag, cp->data))
return 1;
if (tag->tag_type == gcide_content_top ||
tag->tag_type == gcide_content_tag) {
dico_list_iterate(tag->v.tag.taglist, inorder_helper, data);
if (cp->fun(1, tag, cp->data))
return 1;
}
return 0;
}
int
gcide_parse_tree_inorder(struct gcide_parse_tree *tp,
int (*fun)(int, struct gcide_tag *, void *),
void *data)
{
struct walk_closure clos;
clos.fun = fun;
clos.data = data;
return inorder_helper(tp->root, &clos);
}
static int
tag_fixup(void *item, void *data)
{
struct gcide_tag *tag = item;
char *textspace = data;
switch (tag->tag_type) {
case gcide_content_text:
tag->v.text = textspace + tag->v.textpos;
break;
case gcide_content_top:
case gcide_content_tag:
dico_list_iterate(tag->v.tag.taglist, tag_fixup, textspace);
break;
default:
break;
}
return 0;
}
static char const *block_tags[] = {
"source",
"sn",
"p",
"q",
"rj",
"gcide_quote",
NULL
};
int
gcide_is_block_tag(struct gcide_tag *tag)
{
if (tag && tag->tag_type == gcide_content_tag) {
char const **p;
for (p = block_tags; *p; p++)
if (strcmp(tag->tag_name, *p) == 0)
return 1;
}
return 0;
}
int
gcide_is_ws_text(struct gcide_tag *tag)
{
if (tag && tag->tag_type == gcide_content_text) {
char *p;
for (p = tag->v.text; *p == '\n' || *p == ' ' || *p == '\t'; p++);
return *p == 0;
}
return 0;
}
/*
* Fix up the nl/br usage. See the comment to ws_fixup, below.
*/
static int
ws_reduce(void *a, void *b, void *data)
{
struct gcide_tag *atag = a;
struct gcide_tag *btag = b;
if (atag->delete)
return 1;
switch (atag->tag_type) {
case gcide_content_tag:
if (strcmp(atag->tag_name, "source") == 0 &&
btag &&
btag->tag_type == gcide_content_text &&
btag->v.text[0] == ']')
btag->v.text++;
dico_list_reduce(atag->v.tag.taglist, ws_reduce, NULL);
if (gcide_is_block_tag(atag) && gcide_is_ws_text(btag))
btag->delete = 1;
break;
case gcide_content_text:
if (atag->v.text[0] == 0)
return 1;
else if (gcide_is_ws_text(atag) && gcide_is_block_tag(btag))
return 1;
else {
size_t len = strlen(atag->v.text);
if (atag->v.text[len-1] == '[' &&
btag &&
btag->tag_type == gcide_content_tag &&
strcmp(btag->tag_name, "source") == 0) {
if (len == 1)
return 1;
atag->v.text[len-1] = 0;
}
}
break;
case gcide_content_nl:
if (btag == NULL)
return 1;
switch (btag->tag_type) {
case gcide_content_tag:
return gcide_is_block_tag(btag);
case gcide_content_text:
if (btag->v.text[0] == ' ' || btag->v.text[0] == '\t')
return 1;
break;
case gcide_content_nl:
case gcide_content_br:
return 1;
default:
break;
}
break;
case gcide_content_br:
if (btag != NULL) {
if (btag->tag_type == gcide_content_nl)
btag->delete = 1;
}
break;
default:
break;
}
return 0;
}
/*
* This analyzes the placement of gcide_content_nl and gcide_content_br
* elements and removes the former where they are superfluous.
* The gcide_content_nl elements are removed when they appear at either
* side of a gcide_content_br element, or before a block tag, or a text
* segment that begins with a whitespace character.
*
* Additionally, curly brackets surrounding a ... construct
* are removed as well.
*/
static void
ws_fixup(struct gcide_tag *tag)
{
if (tag->tag_type == gcide_content_top ||
tag->tag_type == gcide_content_tag)
dico_list_reduce(tag->v.tag.taglist, ws_reduce, NULL);
}
static size_t
greek_translit(size_t n)
{
while (textspace[n]) {
size_t rd;
const char *greek = gcide_grk_to_utf8(textspace + n, &rd);
if (greek) {
text_add_str(greek, strlen(greek));
n += rd;
} else {
text_add_chr(textspace[n]);
n++;
}
}
return text_segment_finish();
}
static int
early_fixup(void *item, void *data)
{
struct gcide_tag *tag = item;
int translate = data ? *(int*)data : 0;
switch (tag->tag_type) {
case gcide_content_text:
if (translate)
tag->v.textpos = greek_translit(tag->v.textpos);
break;
case gcide_content_top:
dico_list_iterate(tag->v.tag.taglist, early_fixup, &translate);
break;
case gcide_content_tag:
translate = gcide_is_tag(tag, "grk");
dico_list_iterate(tag->v.tag.taglist, early_fixup, &translate);
break;
default:
break;
}
return 0;
}
/*
* The blocks in GCIDE corpus often begin with "as, ". This looks strange
* if another font is used for content or if it is otherwise marked up.
* To avoid this, move "as, " out of block.
*/
static void
as_fixup(struct gcide_tag *tag)
{
dico_iterator_t itr;
switch (tag->tag_type) {
case gcide_content_top:
case gcide_content_tag:
itr = dico_list_iterator(tag->v.tag.taglist);
tag = dico_iterator_first(itr);
while (tag) {
if (gcide_is_tag(tag, "as")) {
struct gcide_tag *t = dico_list_head(tag->v.tag.taglist);
char *s = textspace + t->v.textpos;
if (strncmp(s, "as", 2) == 0 &&
(isspace(s[3]) || ispunct(s[3]))) {
int len;
struct gcide_tag *t1;
for (len = 3; s[len] && isspace(s[len]); len++);
text_add_str(s, len);
t1 = alloc_tag(gcide_content_text);
t1->v.textpos = text_segment_finish();
dico_iterator_insert_before(itr, t1);
dico_list_remove(tag->v.tag.taglist, t, NULL);
if (s[len]) {
t1 = alloc_tag(gcide_content_text);
text_add_strz(s + len);
t1->v.textpos = text_segment_finish();
dico_list_prepend(tag->v.tag.taglist, t1);
}
}
} else
as_fixup(tag);
tag = dico_iterator_next(itr);
}
dico_iterator_destroy(&itr);
break;
default:
break;
}
}
struct gcide_parse_tree *
gcide_markup_parse(char const *text, size_t len, int dbg, struct gcide_locus *loc)
{
struct gcide_parse_tree *tp;
struct gcide_tag *p;
input_buf = text;
input_len = len;
token_beg = token_end = 0;
first_p_seen = 0;
base_locus = loc;
if (setjmp(errbuf)) {
free(textspace);
dico_list_destroy(&tagstk);
free_tag(current_tag, NULL);
}
if (SIZE_MAX / 2 < len)
memerr("gcide_markup_parse");
textsize = 2 * len;
textspace = malloc(textsize);
if (!textspace)
memerr("gcide_markup_parse");
textpos = textstart = 0;
tagstk = dico_list_create();
if (!tagstk)
memerr("gcide_markup_parse");
dico_list_set_free_item(tagstk, free_tag, NULL);
current_tag = alloc_tag(gcide_content_top);
yy_flex_debug = dbg;
BEGIN(INITIAL);
while (yylex())
;
/* Append trailing text segment, if any */
flush_text_segment();
if (current_tag->tag_type != gcide_content_top) {
p = current_tag;
do {
if (p->tag_type == gcide_content_tag)
dico_log(L_WARN, 0, "%s:%zu: unclosed %s tag",
LOCUS_FILE, LOCUS_OFF(p->offset), p->tag_name);
current_tag = p;
} while ((p = dico_list_pop(tagstk)));
}
dico_list_destroy(&tagstk);
if (!current_tag) {
free(textspace);
return NULL;
}
early_fixup(current_tag, NULL);
as_fixup(current_tag);
tp = malloc(sizeof(*tp));
if (!tp)
memerr("gcide_markup_parse");
tp->textspace = textspace;
tp->textsize = textsize;
tp->root = current_tag;
tag_fixup(tp->root, textspace);
ws_fixup(tp->root);
return tp;
}
void
gcide_parse_tree_free(struct gcide_parse_tree *tp)
{
free(tp->textspace);
free_tag(tp->root, NULL);
free(tp);
}