| 1 |
/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ |
| 2 |
/* ==================================================================== |
| 3 |
* Copyright (c) 1999-2007 Carnegie Mellon University. All rights |
| 4 |
* reserved. |
| 5 |
* |
| 6 |
* Redistribution and use in source and binary forms, with or without |
| 7 |
* modification, are permitted provided that the following conditions |
| 8 |
* are met: |
| 9 |
* |
| 10 |
* 1. Redistributions of source code must retain the above copyright |
| 11 |
* notice, this list of conditions and the following disclaimer. |
| 12 |
* |
| 13 |
* 2. Redistributions in binary form must reproduce the above copyright |
| 14 |
* notice, this list of conditions and the following disclaimer in |
| 15 |
* the documentation and/or other materials provided with the |
| 16 |
* distribution. |
| 17 |
* |
| 18 |
* This work was supported in part by funding from the Defense Advanced |
| 19 |
* Research Projects Agency and the National Science Foundation of the |
| 20 |
* United States of America, and the CMU Sphinx Speech Consortium. |
| 21 |
* |
| 22 |
* THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND |
| 23 |
* ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, |
| 24 |
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
| 25 |
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY |
| 26 |
* NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 27 |
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 28 |
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 29 |
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 30 |
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 31 |
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 32 |
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 33 |
* |
| 34 |
* ==================================================================== |
| 35 |
* |
| 36 |
*/ |
| 37 |
/* |
| 38 |
* \file ngram_model.c N-Gram language models. |
| 39 |
* |
| 40 |
* Author: David Huggins-Daines, much code taken from sphinx3/src/libs3decoder/liblm |
| 41 |
*/ |
| 42 |
|
| 43 |
#ifdef HAVE_CONFIG_H |
| 44 |
#include <config.h> |
| 45 |
#endif |
| 46 |
|
| 47 |
#include <string.h> |
| 48 |
#include <assert.h> |
| 49 |
|
| 50 |
#include "sphinxbase/ngram_model.h" |
| 51 |
#include "sphinxbase/ckd_alloc.h" |
| 52 |
#include "sphinxbase/filename.h" |
| 53 |
#include "sphinxbase/pio.h" |
| 54 |
#include "sphinxbase/err.h" |
| 55 |
#include "sphinxbase/logmath.h" |
| 56 |
#include "sphinxbase/strfuncs.h" |
| 57 |
#include "sphinxbase/case.h" |
| 58 |
|
| 59 |
#include "ngram_model_internal.h" |
| 60 |
#include "ngram_model_trie.h" |
| 61 |
|
| 62 |
ngram_file_type_t |
| 63 |
ngram_file_name_to_type(const char *file_name) |
| 64 |
{ |
| 65 |
const char *ext; |
| 66 |
|
| 67 |
ext = strrchr(file_name, '.'); |
| 68 |
if (ext == NULL) { |
| 69 |
return NGRAM_INVALID; |
| 70 |
} |
| 71 |
if (0 == strcmp_nocase(ext, ".gz")) { |
| 72 |
while (--ext >= file_name) { |
| 73 |
if (*ext == '.') |
| 74 |
break; |
| 75 |
} |
| 76 |
if (ext < file_name) { |
| 77 |
return NGRAM_INVALID; |
| 78 |
} |
| 79 |
} |
| 80 |
else if (0 == strcmp_nocase(ext, ".bz2")) { |
| 81 |
while (--ext >= file_name) { |
| 82 |
if (*ext == '.') |
| 83 |
break; |
| 84 |
} |
| 85 |
if (ext < file_name) { |
| 86 |
return NGRAM_INVALID; |
| 87 |
} |
| 88 |
} |
| 89 |
/* We use strncmp because there might be a .gz on the end. */ |
| 90 |
if (0 == strncmp_nocase(ext, ".ARPA", 5)) |
| 91 |
return NGRAM_ARPA; |
| 92 |
if (0 == strncmp_nocase(ext, ".DMP", 4) |
| 93 |
|| 0 == strncmp_nocase(ext, ".BIN", 4)) |
| 94 |
return NGRAM_BIN; |
| 95 |
return NGRAM_INVALID; |
| 96 |
} |
| 97 |
|
| 98 |
ngram_file_type_t |
| 99 |
ngram_str_to_type(const char *str_name) |
| 100 |
{ |
| 101 |
if (0 == strcmp_nocase(str_name, "arpa")) |
| 102 |
return NGRAM_ARPA; |
| 103 |
if (0 == strcmp_nocase(str_name, "dmp") |
| 104 |
|| 0 == strcmp_nocase(str_name, "bin")) |
| 105 |
return NGRAM_BIN; |
| 106 |
return NGRAM_INVALID; |
| 107 |
} |
| 108 |
|
| 109 |
char const * |
| 110 |
ngram_type_to_str(int type) |
| 111 |
{ |
| 112 |
switch (type) { |
| 113 |
case NGRAM_ARPA: |
| 114 |
return "arpa"; |
| 115 |
case NGRAM_BIN: |
| 116 |
return "dmp/bin"; |
| 117 |
default: |
| 118 |
return NULL; |
| 119 |
} |
| 120 |
} |
| 121 |
|
| 122 |
|
| 123 |
ngram_model_t * |
| 124 |
ngram_model_read(cmd_ln_t * config, |
| 125 |
const char *file_name, |
| 126 |
ngram_file_type_t file_type, logmath_t * lmath) |
| 127 |
{ |
| 128 |
ngram_model_t *model = NULL; |
| 129 |
switch (file_type) { |
| 130 |
case NGRAM_AUTO:{ |
| 131 |
if ((model = |
| 132 |
ngram_model_trie_read_bin(config, file_name, |
| 133 |
lmath)) != NULL) |
| 134 |
break; |
| 135 |
if ((model = |
| 136 |
ngram_model_trie_read_arpa(config, file_name, |
| 137 |
lmath)) != NULL) |
| 138 |
break; |
| 139 |
if ((model = |
| 140 |
ngram_model_trie_read_dmp(config, file_name, |
| 141 |
lmath)) != NULL) |
| 142 |
break; |
| 143 |
return NULL; |
| 144 |
} |
| 145 |
case NGRAM_ARPA: |
| 146 |
model = ngram_model_trie_read_arpa(config, file_name, lmath); |
| 147 |
break; |
| 148 |
case NGRAM_BIN: |
| 149 |
if ((model = |
| 150 |
ngram_model_trie_read_bin(config, file_name, lmath)) != NULL) |
| 151 |
break; |
| 152 |
if ((model = |
| 153 |
ngram_model_trie_read_dmp(config, file_name, lmath)) != NULL) |
| 154 |
break; |
| 155 |
return NULL; |
| 156 |
default: |
| 157 |
E_ERROR("language model file type not supported\n"); |
| 158 |
return NULL; |
| 159 |
} |
| 160 |
|
| 161 |
/* Now set weights based on config if present. */ |
| 162 |
if (config) { |
| 163 |
float32 lw = 1.0; |
| 164 |
float32 wip = 1.0; |
| 165 |
|
| 166 |
if (cmd_ln_exists_r(config, "-lw")) |
| 167 |
lw = cmd_ln_float32_r(config, "-lw"); |
| 168 |
if (cmd_ln_exists_r(config, "-wip")) |
| 169 |
wip = cmd_ln_float32_r(config, "-wip"); |
| 170 |
|
| 171 |
ngram_model_apply_weights(model, lw, wip); |
| 172 |
} |
| 173 |
|
| 174 |
return model; |
| 175 |
} |
| 176 |
|
| 177 |
int |
| 178 |
ngram_model_write(ngram_model_t * model, const char *file_name, |
| 179 |
ngram_file_type_t file_type) |
| 180 |
{ |
| 181 |
switch (file_type) { |
| 182 |
case NGRAM_AUTO:{ |
| 183 |
file_type = ngram_file_name_to_type(file_name); |
| 184 |
/* Default to ARPA (catches .lm and other things) */ |
| 185 |
if (file_type == NGRAM_INVALID) |
| 186 |
file_type = NGRAM_ARPA; |
| 187 |
return ngram_model_write(model, file_name, file_type); |
| 188 |
} |
| 189 |
case NGRAM_ARPA: |
| 190 |
return ngram_model_trie_write_arpa(model, file_name); |
| 191 |
case NGRAM_BIN: |
| 192 |
return ngram_model_trie_write_bin(model, file_name); |
| 193 |
default: |
| 194 |
E_ERROR("language model file type not supported\n"); |
| 195 |
return -1; |
| 196 |
} |
| 197 |
E_ERROR("language model file type not supported\n"); |
| 198 |
return -1; |
| 199 |
} |
| 200 |
|
| 201 |
int32 |
| 202 |
ngram_model_init(ngram_model_t * base, |
| 203 |
ngram_funcs_t * funcs, |
| 204 |
logmath_t * lmath, int32 n, int32 n_unigram) |
| 205 |
{ |
| 206 |
base->refcount = 1; |
| 207 |
base->funcs = funcs; |
| 208 |
base->n = n; |
| 209 |
/* If this was previously initialized... */ |
| 210 |
if (base->n_counts == NULL) |
| 211 |
base->n_counts = (uint32 *) ckd_calloc(n, sizeof(*base->n_counts)); |
| 212 |
/* Don't reset weights if logmath object hasn't changed. */ |
| 213 |
if (base->lmath != lmath) { |
| 214 |
/* Set default values for weights. */ |
| 215 |
base->lw = 1.0; |
| 216 |
base->log_wip = 0; /* i.e. 1.0 */ |
| 217 |
base->log_zero = logmath_get_zero(lmath); |
| 218 |
base->lmath = lmath; |
| 219 |
} |
| 220 |
/* Allocate or reallocate space for word strings. */ |
| 221 |
if (base->word_str) { |
| 222 |
/* Free all previous word strings if they were allocated. */ |
| 223 |
if (base->writable) { |
| 224 |
int32 i; |
| 225 |
for (i = 0; i < base->n_words; ++i) { |
| 226 |
ckd_free(base->word_str[i]); |
| 227 |
base->word_str[i] = NULL; |
| 228 |
} |
| 229 |
} |
| 230 |
base->word_str = |
| 231 |
(char **) ckd_realloc(base->word_str, |
| 232 |
n_unigram * sizeof(char *)); |
| 233 |
} |
| 234 |
else { |
| 235 |
base->word_str = (char **) ckd_calloc(n_unigram, sizeof(char *)); |
| 236 |
} |
| 237 |
/* NOTE: They are no longer case-insensitive since we are allowing |
| 238 |
* other encodings for word strings. Beware. */ |
| 239 |
if (base->wid) |
| 240 |
hash_table_empty(base->wid); |
| 241 |
else |
| 242 |
base->wid = hash_table_new(n_unigram, FALSE); |
| 243 |
base->n_counts[0] = base->n_1g_alloc = base->n_words = n_unigram; |
| 244 |
|
| 245 |
return 0; |
| 246 |
} |
| 247 |
|
| 248 |
ngram_model_t * |
| 249 |
ngram_model_retain(ngram_model_t * model) |
| 250 |
{ |
| 251 |
++model->refcount; |
| 252 |
return model; |
| 253 |
} |
| 254 |
|
| 255 |
void |
| 256 |
ngram_model_flush(ngram_model_t * model) |
| 257 |
{ |
| 258 |
if (model->funcs && model->funcs->flush) |
| 259 |
(*model->funcs->flush) (model); |
| 260 |
} |
| 261 |
|
| 262 |
int |
| 263 |
ngram_model_free(ngram_model_t * model) |
| 264 |
{ |
| 265 |
int i; |
| 266 |
|
| 267 |
if (model == NULL) |
| 268 |
return 0; |
| 269 |
if (--model->refcount > 0) |
| 270 |
return model->refcount; |
| 271 |
if (model->funcs && model->funcs->free) |
| 272 |
(*model->funcs->free) (model); |
| 273 |
if (model->writable) { |
| 274 |
/* Free all words. */ |
| 275 |
for (i = 0; i < model->n_words; ++i) { |
| 276 |
ckd_free(model->word_str[i]); |
| 277 |
} |
| 278 |
} |
| 279 |
else { |
| 280 |
/* Free all class words. */ |
| 281 |
for (i = 0; i < model->n_classes; ++i) { |
| 282 |
ngram_class_t *lmclass; |
| 283 |
int32 j; |
| 284 |
|
| 285 |
lmclass = model->classes[i]; |
| 286 |
for (j = 0; j < lmclass->n_words; ++j) { |
| 287 |
ckd_free(model->word_str[lmclass->start_wid + j]); |
| 288 |
} |
| 289 |
for (j = 0; j < lmclass->n_hash; ++j) { |
| 290 |
if (lmclass->nword_hash[j].wid != -1) { |
| 291 |
ckd_free(model->word_str[lmclass->nword_hash[j].wid]); |
| 292 |
} |
| 293 |
} |
| 294 |
} |
| 295 |
} |
| 296 |
for (i = 0; i < model->n_classes; ++i) { |
| 297 |
ngram_class_free(model->classes[i]); |
| 298 |
} |
| 299 |
ckd_free(model->classes); |
| 300 |
hash_table_free(model->wid); |
| 301 |
ckd_free(model->word_str); |
| 302 |
ckd_free(model->n_counts); |
| 303 |
ckd_free(model); |
| 304 |
return 0; |
| 305 |
} |
| 306 |
|
| 307 |
int |
| 308 |
ngram_model_casefold(ngram_model_t * model, int kase) |
| 309 |
{ |
| 310 |
int writable, i; |
| 311 |
hash_table_t *new_wid; |
| 312 |
|
| 313 |
/* Were word strings already allocated? */ |
| 314 |
writable = model->writable; |
| 315 |
/* Either way, we are going to allocate some word strings. */ |
| 316 |
model->writable = TRUE; |
| 317 |
|
| 318 |
/* And, don't forget, we need to rebuild the word to unigram ID |
| 319 |
* mapping. */ |
| 320 |
new_wid = hash_table_new(model->n_words, FALSE); |
| 321 |
for (i = 0; i < model->n_words; ++i) { |
| 322 |
char *outstr; |
| 323 |
if (writable) { |
| 324 |
outstr = model->word_str[i]; |
| 325 |
} |
| 326 |
else { |
| 327 |
outstr = ckd_salloc(model->word_str[i]); |
| 328 |
} |
| 329 |
/* Don't case-fold <tags> or [classes] */ |
| 330 |
if (outstr[0] == '<' || outstr[0] == '[') { |
| 331 |
} |
| 332 |
else { |
| 333 |
switch (kase) { |
| 334 |
case NGRAM_UPPER: |
| 335 |
ucase(outstr); |
| 336 |
break; |
| 337 |
case NGRAM_LOWER: |
| 338 |
lcase(outstr); |
| 339 |
break; |
| 340 |
default: |
| 341 |
; |
| 342 |
} |
| 343 |
} |
| 344 |
model->word_str[i] = outstr; |
| 345 |
|
| 346 |
/* Now update the hash table. We might have terrible |
| 347 |
* collisions here, so warn about them. */ |
| 348 |
if (hash_table_enter_int32(new_wid, model->word_str[i], i) != i) { |
| 349 |
E_WARN("Duplicate word in dictionary after conversion: %s\n", |
| 350 |
model->word_str[i]); |
| 351 |
} |
| 352 |
} |
| 353 |
/* Swap out the hash table. */ |
| 354 |
hash_table_free(model->wid); |
| 355 |
model->wid = new_wid; |
| 356 |
return 0; |
| 357 |
} |
| 358 |
|
| 359 |
int |
| 360 |
ngram_model_apply_weights(ngram_model_t * model, float32 lw, float32 wip) |
| 361 |
{ |
| 362 |
return (*model->funcs->apply_weights) (model, lw, wip); |
| 363 |
} |
| 364 |
|
| 365 |
float32 |
| 366 |
ngram_model_get_weights(ngram_model_t * model, int32 * out_log_wip) |
| 367 |
{ |
| 368 |
if (out_log_wip) |
| 369 |
*out_log_wip = model->log_wip; |
| 370 |
return model->lw; |
| 371 |
} |
| 372 |
|
| 373 |
|
| 374 |
int32 |
| 375 |
ngram_ng_score(ngram_model_t * model, int32 wid, int32 * history, |
| 376 |
int32 n_hist, int32 * n_used) |
| 377 |
{ |
| 378 |
int32 score, class_weight = 0; |
| 379 |
int i; |
| 380 |
|
| 381 |
/* Closed vocabulary, OOV word probability is zero */ |
| 382 |
if (wid == NGRAM_INVALID_WID) |
| 383 |
return model->log_zero; |
| 384 |
|
| 385 |
/* "Declassify" wid and history */ |
| 386 |
if (NGRAM_IS_CLASSWID(wid)) { |
| 387 |
ngram_class_t *lmclass = model->classes[NGRAM_CLASSID(wid)]; |
| 388 |
|
| 389 |
class_weight = ngram_class_prob(lmclass, wid); |
| 390 |
if (class_weight == 1) /* Meaning, not found in class. */ |
| 391 |
return model->log_zero; |
| 392 |
wid = lmclass->tag_wid; |
| 393 |
} |
| 394 |
for (i = 0; i < n_hist; ++i) { |
| 395 |
if (history[i] != NGRAM_INVALID_WID |
| 396 |
&& NGRAM_IS_CLASSWID(history[i])) |
| 397 |
history[i] = |
| 398 |
model->classes[NGRAM_CLASSID(history[i])]->tag_wid; |
| 399 |
} |
| 400 |
score = (*model->funcs->score) (model, wid, history, n_hist, n_used); |
| 401 |
|
| 402 |
/* Multiply by unigram in-class weight. */ |
| 403 |
return score + class_weight; |
| 404 |
} |
| 405 |
|
| 406 |
int32 |
| 407 |
ngram_score(ngram_model_t * model, const char *word, ...) |
| 408 |
{ |
| 409 |
va_list history; |
| 410 |
const char *hword; |
| 411 |
int32 *histid; |
| 412 |
int32 n_hist; |
| 413 |
int32 n_used; |
| 414 |
int32 prob; |
| 415 |
|
| 416 |
va_start(history, word); |
| 417 |
n_hist = 0; |
| 418 |
while ((hword = va_arg(history, const char *)) != NULL) |
| 419 |
++n_hist; |
| 420 |
va_end(history); |
| 421 |
|
| 422 |
histid = ckd_calloc(n_hist, sizeof(*histid)); |
| 423 |
va_start(history, word); |
| 424 |
n_hist = 0; |
| 425 |
while ((hword = va_arg(history, const char *)) != NULL) { |
| 426 |
histid[n_hist] = ngram_wid(model, hword); |
| 427 |
++n_hist; |
| 428 |
} |
| 429 |
va_end(history); |
| 430 |
|
| 431 |
prob = ngram_ng_score(model, ngram_wid(model, word), |
| 432 |
histid, n_hist, &n_used); |
| 433 |
ckd_free(histid); |
| 434 |
return prob; |
| 435 |
} |
| 436 |
|
| 437 |
int32 |
| 438 |
ngram_tg_score(ngram_model_t * model, int32 w3, int32 w2, int32 w1, |
| 439 |
int32 * n_used) |
| 440 |
{ |
| 441 |
int32 hist[2]; |
| 442 |
hist[0] = w2; |
| 443 |
hist[1] = w1; |
| 444 |
return ngram_ng_score(model, w3, hist, 2, n_used); |
| 445 |
} |
| 446 |
|
| 447 |
int32 |
| 448 |
ngram_bg_score(ngram_model_t * model, int32 w2, int32 w1, int32 * n_used) |
| 449 |
{ |
| 450 |
return ngram_ng_score(model, w2, &w1, 1, n_used); |
| 451 |
} |
| 452 |
|
| 453 |
int32 |
| 454 |
ngram_ng_prob(ngram_model_t * model, int32 wid, int32 * history, |
| 455 |
int32 n_hist, int32 * n_used) |
| 456 |
{ |
| 457 |
int32 prob, class_weight = 0; |
| 458 |
int i; |
| 459 |
|
| 460 |
/* Closed vocabulary, OOV word probability is zero */ |
| 461 |
if (wid == NGRAM_INVALID_WID) |
| 462 |
return model->log_zero; |
| 463 |
|
| 464 |
/* "Declassify" wid and history */ |
| 465 |
if (NGRAM_IS_CLASSWID(wid)) { |
| 466 |
ngram_class_t *lmclass = model->classes[NGRAM_CLASSID(wid)]; |
| 467 |
|
| 468 |
class_weight = ngram_class_prob(lmclass, wid); |
| 469 |
if (class_weight == 1) /* Meaning, not found in class. */ |
| 470 |
return class_weight; |
| 471 |
wid = lmclass->tag_wid; |
| 472 |
} |
| 473 |
for (i = 0; i < n_hist; ++i) { |
| 474 |
if (history[i] != NGRAM_INVALID_WID |
| 475 |
&& NGRAM_IS_CLASSWID(history[i])) |
| 476 |
history[i] = |
| 477 |
model->classes[NGRAM_CLASSID(history[i])]->tag_wid; |
| 478 |
} |
| 479 |
prob = (*model->funcs->raw_score) (model, wid, history, |
| 480 |
n_hist, n_used); |
| 481 |
/* Multiply by unigram in-class weight. */ |
| 482 |
return prob + class_weight; |
| 483 |
} |
| 484 |
|
| 485 |
int32 |
| 486 |
ngram_probv(ngram_model_t * model, const char *word, ...) |
| 487 |
{ |
| 488 |
va_list history; |
| 489 |
const char *hword; |
| 490 |
int32 *histid; |
| 491 |
int32 n_hist; |
| 492 |
int32 n_used; |
| 493 |
int32 prob; |
| 494 |
|
| 495 |
va_start(history, word); |
| 496 |
n_hist = 0; |
| 497 |
while ((hword = va_arg(history, const char *)) != NULL) |
| 498 |
++n_hist; |
| 499 |
va_end(history); |
| 500 |
|
| 501 |
histid = ckd_calloc(n_hist, sizeof(*histid)); |
| 502 |
va_start(history, word); |
| 503 |
n_hist = 0; |
| 504 |
while ((hword = va_arg(history, const char *)) != NULL) { |
| 505 |
histid[n_hist] = ngram_wid(model, hword); |
| 506 |
++n_hist; |
| 507 |
} |
| 508 |
va_end(history); |
| 509 |
|
| 510 |
prob = ngram_ng_prob(model, ngram_wid(model, word), |
| 511 |
histid, n_hist, &n_used); |
| 512 |
ckd_free(histid); |
| 513 |
return prob; |
| 514 |
} |
| 515 |
|
| 516 |
int32 |
| 517 |
ngram_prob(ngram_model_t * model, const char* const *words, int32 n) |
| 518 |
{ |
| 519 |
int32 *ctx_id; |
| 520 |
int32 nused; |
| 521 |
int32 prob; |
| 522 |
int32 wid; |
| 523 |
uint32 i; |
| 524 |
|
| 525 |
ctx_id = (int32 *) ckd_calloc(n - 1, sizeof(*ctx_id)); |
| 526 |
for (i = 1; i < (uint32) n; ++i) |
| 527 |
ctx_id[i - 1] = ngram_wid(model, words[i]); |
| 528 |
|
| 529 |
wid = ngram_wid(model, *words); |
| 530 |
prob = ngram_ng_prob(model, wid, ctx_id, n - 1, &nused); |
| 531 |
ckd_free(ctx_id); |
| 532 |
|
| 533 |
return prob; |
| 534 |
} |
| 535 |
|
| 536 |
int32 |
| 537 |
ngram_score_to_prob(ngram_model_t * base, int32 score) |
| 538 |
{ |
| 539 |
int32 prob; |
| 540 |
|
| 541 |
/* Undo insertion penalty. */ |
| 542 |
prob = score - base->log_wip; |
| 543 |
/* Undo language weight. */ |
| 544 |
prob = (int32) (prob / base->lw); |
| 545 |
|
| 546 |
return prob; |
| 547 |
} |
| 548 |
|
| 549 |
int32 |
| 550 |
ngram_unknown_wid(ngram_model_t * model) |
| 551 |
{ |
| 552 |
int32 val; |
| 553 |
|
| 554 |
/* FIXME: This could be memoized for speed if necessary. */ |
| 555 |
/* Look up <UNK>, if not found return NGRAM_INVALID_WID. */ |
| 556 |
if (hash_table_lookup_int32(model->wid, "<UNK>", &val) == -1) |
| 557 |
return NGRAM_INVALID_WID; |
| 558 |
else |
| 559 |
return val; |
| 560 |
} |
| 561 |
|
| 562 |
int32 |
| 563 |
ngram_zero(ngram_model_t * model) |
| 564 |
{ |
| 565 |
return model->log_zero; |
| 566 |
} |
| 567 |
|
| 568 |
int32 |
| 569 |
ngram_model_get_size(ngram_model_t * model) |
| 570 |
{ |
| 571 |
if (model != NULL) |
| 572 |
return model->n; |
| 573 |
return 0; |
| 574 |
} |
| 575 |
|
| 576 |
uint32 const * |
| 577 |
ngram_model_get_counts(ngram_model_t * model) |
| 578 |
{ |
| 579 |
if (model != NULL) |
| 580 |
return model->n_counts; |
| 581 |
return NULL; |
| 582 |
} |
| 583 |
|
| 584 |
int32 |
| 585 |
ngram_wid(ngram_model_t * model, const char *word) |
| 586 |
{ |
| 587 |
int32 val; |
| 588 |
|
| 589 |
if (hash_table_lookup_int32(model->wid, word, &val) == -1) |
| 590 |
return ngram_unknown_wid(model); |
| 591 |
else |
| 592 |
return val; |
| 593 |
} |
| 594 |
|
| 595 |
const char * |
| 596 |
ngram_word(ngram_model_t * model, int32 wid) |
| 597 |
{ |
| 598 |
/* Remove any class tag */ |
| 599 |
wid = NGRAM_BASEWID(wid); |
| 600 |
if (wid >= model->n_words) |
| 601 |
return NULL; |
| 602 |
return model->word_str[wid]; |
| 603 |
} |
| 604 |
|
| 605 |
/** |
| 606 |
* Add a word to the word string and ID mapping. |
| 607 |
*/ |
| 608 |
int32 |
| 609 |
ngram_add_word_internal(ngram_model_t * model, |
| 610 |
const char *word, int32 classid) |
| 611 |
{ |
| 612 |
|
| 613 |
/* Check for hash collisions. */ |
| 614 |
int32 wid; |
| 615 |
if (hash_table_lookup_int32(model->wid, word, &wid) == 0) { |
| 616 |
E_WARN("Omit duplicate word '%s'\n", word); |
| 617 |
return wid; |
| 618 |
} |
| 619 |
|
| 620 |
/* Take the next available word ID */ |
| 621 |
wid = model->n_words; |
| 622 |
if (classid >= 0) { |
| 623 |
wid = NGRAM_CLASSWID(wid, classid); |
| 624 |
} |
| 625 |
|
| 626 |
/* Reallocate word_str if necessary. */ |
| 627 |
if (model->n_words >= model->n_1g_alloc) { |
| 628 |
model->n_1g_alloc += UG_ALLOC_STEP; |
| 629 |
model->word_str = ckd_realloc(model->word_str, |
| 630 |
sizeof(*model->word_str) * |
| 631 |
model->n_1g_alloc); |
| 632 |
} |
| 633 |
/* Add the word string in the appropriate manner. */ |
| 634 |
/* Class words are always dynamically allocated. */ |
| 635 |
model->word_str[model->n_words] = ckd_salloc(word); |
| 636 |
/* Now enter it into the hash table. */ |
| 637 |
if (hash_table_enter_int32 |
| 638 |
(model->wid, model->word_str[model->n_words], wid) != wid) { |
| 639 |
E_ERROR |
| 640 |
("Hash insertion failed for word %s => %p (should not happen)\n", |
| 641 |
model->word_str[model->n_words], (void *) (long) (wid)); |
| 642 |
} |
| 643 |
/* Increment number of words. */ |
| 644 |
++model->n_words; |
| 645 |
return wid; |
| 646 |
} |
| 647 |
|
| 648 |
int32 |
| 649 |
ngram_model_add_word(ngram_model_t * model, |
| 650 |
const char *word, float32 weight) |
| 651 |
{ |
| 652 |
int32 wid, prob = model->log_zero; |
| 653 |
|
| 654 |
/* If we add word to unwritable model, we need to make it writable */ |
| 655 |
if (!model->writable) { |
| 656 |
E_WARN("Can't add word '%s' to read-only language model. " |
| 657 |
"Disable mmap with '-mmap no' to make it writable\n", word); |
| 658 |
return -1; |
| 659 |
} |
| 660 |
|
| 661 |
wid = ngram_add_word_internal(model, word, -1); |
| 662 |
if (wid == NGRAM_INVALID_WID) |
| 663 |
return wid; |
| 664 |
|
| 665 |
/* Do what needs to be done to add the word to the unigram. */ |
| 666 |
if (model->funcs && model->funcs->add_ug) |
| 667 |
prob = |
| 668 |
(*model->funcs->add_ug) (model, wid, |
| 669 |
logmath_log(model->lmath, weight)); |
| 670 |
if (prob == 0) |
| 671 |
return -1; |
| 672 |
|
| 673 |
return wid; |
| 674 |
} |
| 675 |
|
| 676 |
ngram_class_t * |
| 677 |
ngram_class_new(ngram_model_t * model, int32 tag_wid, int32 start_wid, |
| 678 |
glist_t classwords) |
| 679 |
{ |
| 680 |
ngram_class_t *lmclass; |
| 681 |
gnode_t *gn; |
| 682 |
float32 tprob; |
| 683 |
int i; |
| 684 |
|
| 685 |
lmclass = ckd_calloc(1, sizeof(*lmclass)); |
| 686 |
lmclass->tag_wid = tag_wid; |
| 687 |
/* wid_base is the wid (minus class tag) of the first word in the list. */ |
| 688 |
lmclass->start_wid = start_wid; |
| 689 |
lmclass->n_words = glist_count(classwords); |
| 690 |
lmclass->prob1 = ckd_calloc(lmclass->n_words, sizeof(*lmclass->prob1)); |
| 691 |
lmclass->nword_hash = NULL; |
| 692 |
lmclass->n_hash = 0; |
| 693 |
tprob = 0.0; |
| 694 |
for (gn = classwords; gn; gn = gnode_next(gn)) { |
| 695 |
tprob += gnode_float32(gn); |
| 696 |
} |
| 697 |
if (tprob > 1.1 || tprob < 0.9) { |
| 698 |
E_INFO("Total class probability is %f, will normalize\n", tprob); |
| 699 |
for (gn = classwords; gn; gn = gnode_next(gn)) { |
| 700 |
gn->data.fl /= tprob; |
| 701 |
} |
| 702 |
} |
| 703 |
for (i = 0, gn = classwords; gn; ++i, gn = gnode_next(gn)) { |
| 704 |
lmclass->prob1[i] = logmath_log(model->lmath, gnode_float32(gn)); |
| 705 |
} |
| 706 |
|
| 707 |
return lmclass; |
| 708 |
} |
| 709 |
|
| 710 |
int32 |
| 711 |
ngram_class_add_word(ngram_class_t * lmclass, int32 wid, int32 lweight) |
| 712 |
{ |
| 713 |
int32 hash; |
| 714 |
|
| 715 |
if (lmclass->nword_hash == NULL) { |
| 716 |
/* Initialize everything in it to -1 */ |
| 717 |
lmclass->nword_hash = |
| 718 |
ckd_malloc(NGRAM_HASH_SIZE * sizeof(*lmclass->nword_hash)); |
| 719 |
memset(lmclass->nword_hash, 0xff, |
| 720 |
NGRAM_HASH_SIZE * sizeof(*lmclass->nword_hash)); |
| 721 |
lmclass->n_hash = NGRAM_HASH_SIZE; |
| 722 |
lmclass->n_hash_inuse = 0; |
| 723 |
} |
| 724 |
/* Stupidest possible hash function. This will work pretty well |
| 725 |
* when this function is called repeatedly with contiguous word |
| 726 |
* IDs, though... */ |
| 727 |
hash = wid & (lmclass->n_hash - 1); |
| 728 |
if (lmclass->nword_hash[hash].wid == -1) { |
| 729 |
/* Good, no collision. */ |
| 730 |
lmclass->nword_hash[hash].wid = wid; |
| 731 |
lmclass->nword_hash[hash].prob1 = lweight; |
| 732 |
++lmclass->n_hash_inuse; |
| 733 |
return hash; |
| 734 |
} |
| 735 |
else { |
| 736 |
int32 next; /**< Next available bucket. */ |
| 737 |
/* Collision... Find the end of the hash chain. */ |
| 738 |
while (lmclass->nword_hash[hash].next != -1) |
| 739 |
hash = lmclass->nword_hash[hash].next; |
| 740 |
assert(hash != -1); |
| 741 |
/* Does we has any more bukkit? */ |
| 742 |
if (lmclass->n_hash_inuse == lmclass->n_hash) { |
| 743 |
/* Oh noes! Ok, we makes more. */ |
| 744 |
lmclass->nword_hash = ckd_realloc(lmclass->nword_hash, |
| 745 |
lmclass->n_hash * 2 * |
| 746 |
sizeof(*lmclass-> |
| 747 |
nword_hash)); |
| 748 |
memset(lmclass->nword_hash + lmclass->n_hash, 0xff, |
| 749 |
lmclass->n_hash * sizeof(*lmclass->nword_hash)); |
| 750 |
/* Just use the next allocated one (easy) */ |
| 751 |
next = lmclass->n_hash; |
| 752 |
lmclass->n_hash *= 2; |
| 753 |
} |
| 754 |
else { |
| 755 |
/* Look for any available bucket. We hope this doesn't happen. */ |
| 756 |
for (next = 0; next < lmclass->n_hash; ++next) |
| 757 |
if (lmclass->nword_hash[next].wid == -1) |
| 758 |
break; |
| 759 |
/* This should absolutely not happen. */ |
| 760 |
assert(next != lmclass->n_hash); |
| 761 |
} |
| 762 |
lmclass->nword_hash[next].wid = wid; |
| 763 |
lmclass->nword_hash[next].prob1 = lweight; |
| 764 |
lmclass->nword_hash[hash].next = next; |
| 765 |
++lmclass->n_hash_inuse; |
| 766 |
return next; |
| 767 |
} |
| 768 |
} |
| 769 |
|
| 770 |
void |
| 771 |
ngram_class_free(ngram_class_t * lmclass) |
| 772 |
{ |
| 773 |
ckd_free(lmclass->nword_hash); |
| 774 |
ckd_free(lmclass->prob1); |
| 775 |
ckd_free(lmclass); |
| 776 |
} |
| 777 |
|
| 778 |
int32 |
| 779 |
ngram_model_add_class_word(ngram_model_t * model, |
| 780 |
const char *classname, |
| 781 |
const char *word, float32 weight) |
| 782 |
{ |
| 783 |
ngram_class_t *lmclass; |
| 784 |
int32 classid, tag_wid, wid, i, scale; |
| 785 |
float32 fprob; |
| 786 |
|
| 787 |
/* Find the class corresponding to classname. Linear search |
| 788 |
* probably okay here since there won't be very many classes, and |
| 789 |
* this doesn't have to be fast. */ |
| 790 |
tag_wid = ngram_wid(model, classname); |
| 791 |
if (tag_wid == NGRAM_INVALID_WID) { |
| 792 |
E_ERROR("No such word or class tag: %s\n", classname); |
| 793 |
return tag_wid; |
| 794 |
} |
| 795 |
for (classid = 0; classid < model->n_classes; ++classid) { |
| 796 |
if (model->classes[classid]->tag_wid == tag_wid) |
| 797 |
break; |
| 798 |
} |
| 799 |
/* Hmm, no such class. It's probably not a good idea to create one. */ |
| 800 |
if (classid == model->n_classes) { |
| 801 |
E_ERROR |
| 802 |
("Word %s is not a class tag (call ngram_model_add_class() first)\n", |
| 803 |
classname); |
| 804 |
return NGRAM_INVALID_WID; |
| 805 |
} |
| 806 |
lmclass = model->classes[classid]; |
| 807 |
|
| 808 |
/* Add this word to the model's set of words. */ |
| 809 |
wid = ngram_add_word_internal(model, word, classid); |
| 810 |
if (wid == NGRAM_INVALID_WID) |
| 811 |
return wid; |
| 812 |
|
| 813 |
/* This is the fixed probability of the new word. */ |
| 814 |
fprob = weight * 1.0f / (lmclass->n_words + lmclass->n_hash_inuse + 1); |
| 815 |
/* Now normalize everything else to fit it in. This is |
| 816 |
* accomplished by simply scaling all the other probabilities |
| 817 |
* by (1-fprob). */ |
| 818 |
scale = logmath_log(model->lmath, 1.0 - fprob); |
| 819 |
for (i = 0; i < lmclass->n_words; ++i) |
| 820 |
lmclass->prob1[i] += scale; |
| 821 |
for (i = 0; i < lmclass->n_hash; ++i) |
| 822 |
if (lmclass->nword_hash[i].wid != -1) |
| 823 |
lmclass->nword_hash[i].prob1 += scale; |
| 824 |
|
| 825 |
/* Now add it to the class hash table. */ |
| 826 |
return ngram_class_add_word(lmclass, wid, |
| 827 |
logmath_log(model->lmath, fprob)); |
| 828 |
} |
| 829 |
|
| 830 |
int32 |
| 831 |
ngram_model_add_class(ngram_model_t * model, |
| 832 |
const char *classname, |
| 833 |
float32 classweight, |
| 834 |
char **words, const float32 * weights, int32 n_words) |
| 835 |
{ |
| 836 |
ngram_class_t *lmclass; |
| 837 |
glist_t classwords = NULL; |
| 838 |
int32 i, start_wid = -1; |
| 839 |
int32 classid, tag_wid; |
| 840 |
|
| 841 |
/* Check if classname already exists in model. If not, add it. */ |
| 842 |
if ((tag_wid = |
| 843 |
ngram_wid(model, classname)) == ngram_unknown_wid(model)) { |
| 844 |
tag_wid = ngram_model_add_word(model, classname, classweight); |
| 845 |
if (tag_wid == NGRAM_INVALID_WID) |
| 846 |
return -1; |
| 847 |
} |
| 848 |
|
| 849 |
if (model->n_classes == 128) { |
| 850 |
E_ERROR("Number of classes cannot exceed 128 (sorry)\n"); |
| 851 |
return -1; |
| 852 |
} |
| 853 |
classid = model->n_classes; |
| 854 |
for (i = 0; i < n_words; ++i) { |
| 855 |
int32 wid; |
| 856 |
|
| 857 |
wid = ngram_add_word_internal(model, words[i], classid); |
| 858 |
if (wid == NGRAM_INVALID_WID) |
| 859 |
return -1; |
| 860 |
if (start_wid == -1) |
| 861 |
start_wid = NGRAM_BASEWID(wid); |
| 862 |
classwords = glist_add_float32(classwords, weights[i]); |
| 863 |
} |
| 864 |
classwords = glist_reverse(classwords); |
| 865 |
lmclass = ngram_class_new(model, tag_wid, start_wid, classwords); |
| 866 |
glist_free(classwords); |
| 867 |
if (lmclass == NULL) |
| 868 |
return -1; |
| 869 |
|
| 870 |
++model->n_classes; |
| 871 |
if (model->classes == NULL) |
| 872 |
model->classes = ckd_calloc(1, sizeof(*model->classes)); |
| 873 |
else |
| 874 |
model->classes = ckd_realloc(model->classes, |
| 875 |
model->n_classes * |
| 876 |
sizeof(*model->classes)); |
| 877 |
model->classes[classid] = lmclass; |
| 878 |
return classid; |
| 879 |
} |
| 880 |
|
| 881 |
int32 |
| 882 |
ngram_class_prob(ngram_class_t * lmclass, int32 wid) |
| 883 |
{ |
| 884 |
int32 base_wid = NGRAM_BASEWID(wid); |
| 885 |
|
| 886 |
if (base_wid < lmclass->start_wid |
| 887 |
|| base_wid > lmclass->start_wid + lmclass->n_words) { |
| 888 |
int32 hash; |
| 889 |
|
| 890 |
/* Look it up in the hash table. */ |
| 891 |
hash = wid & (lmclass->n_hash - 1); |
| 892 |
while (hash != -1 && lmclass->nword_hash[hash].wid != wid) |
| 893 |
hash = lmclass->nword_hash[hash].next; |
| 894 |
if (hash == -1) |
| 895 |
return 1; |
| 896 |
return lmclass->nword_hash[hash].prob1; |
| 897 |
} |
| 898 |
else { |
| 899 |
return lmclass->prob1[base_wid - lmclass->start_wid]; |
| 900 |
} |
| 901 |
} |
| 902 |
|
| 903 |
int32 |
| 904 |
read_classdef_file(hash_table_t * classes, const char *file_name) |
| 905 |
{ |
| 906 |
FILE *fp; |
| 907 |
int32 is_pipe; |
| 908 |
int inclass; /**< Are we currently reading a list of class words? */ |
| 909 |
int32 rv = -1; |
| 910 |
gnode_t *gn; |
| 911 |
glist_t classwords = NULL; |
| 912 |
glist_t classprobs = NULL; |
| 913 |
char *classname = NULL; |
| 914 |
|
| 915 |
if ((fp = fopen_comp(file_name, "r", &is_pipe)) == NULL) { |
| 916 |
E_ERROR("File %s not found\n", file_name); |
| 917 |
return -1; |
| 918 |
} |
| 919 |
|
| 920 |
inclass = FALSE; |
| 921 |
while (!feof(fp)) { |
| 922 |
char line[512]; |
| 923 |
char *wptr[2]; |
| 924 |
int n_words; |
| 925 |
|
| 926 |
if (fgets(line, sizeof(line), fp) == NULL) |
| 927 |
break; |
| 928 |
|
| 929 |
n_words = str2words(line, wptr, 2); |
| 930 |
if (n_words <= 0) |
| 931 |
continue; |
| 932 |
|
| 933 |
if (inclass) { |
| 934 |
/* Look for an end of class marker. */ |
| 935 |
if (n_words == 2 && 0 == strcmp(wptr[0], "END")) { |
| 936 |
classdef_t *classdef; |
| 937 |
gnode_t *word, *weight; |
| 938 |
int32 i; |
| 939 |
|
| 940 |
if (classname == NULL || 0 != strcmp(wptr[1], classname)) |
| 941 |
goto error_out; |
| 942 |
inclass = FALSE; |
| 943 |
|
| 944 |
/* Construct a class from the list of words collected. */ |
| 945 |
classdef = ckd_calloc(1, sizeof(*classdef)); |
| 946 |
classwords = glist_reverse(classwords); |
| 947 |
classprobs = glist_reverse(classprobs); |
| 948 |
classdef->n_words = glist_count(classwords); |
| 949 |
classdef->words = ckd_calloc(classdef->n_words, |
| 950 |
sizeof(*classdef->words)); |
| 951 |
classdef->weights = ckd_calloc(classdef->n_words, |
| 952 |
sizeof(*classdef->weights)); |
| 953 |
word = classwords; |
| 954 |
weight = classprobs; |
| 955 |
for (i = 0; i < classdef->n_words; ++i) { |
| 956 |
classdef->words[i] = gnode_ptr(word); |
| 957 |
classdef->weights[i] = gnode_float32(weight); |
| 958 |
word = gnode_next(word); |
| 959 |
weight = gnode_next(weight); |
| 960 |
} |
| 961 |
|
| 962 |
/* Add this class to the hash table. */ |
| 963 |
if (hash_table_enter(classes, classname, classdef) != |
| 964 |
classdef) { |
| 965 |
classdef_free(classdef); |
| 966 |
goto error_out; |
| 967 |
} |
| 968 |
|
| 969 |
/* Reset everything. */ |
| 970 |
glist_free(classwords); |
| 971 |
glist_free(classprobs); |
| 972 |
classwords = NULL; |
| 973 |
classprobs = NULL; |
| 974 |
classname = NULL; |
| 975 |
} |
| 976 |
else { |
| 977 |
float32 fprob; |
| 978 |
|
| 979 |
if (n_words == 2) |
| 980 |
fprob = atof_c(wptr[1]); |
| 981 |
else |
| 982 |
fprob = 1.0f; |
| 983 |
/* Add it to the list of words for this class. */ |
| 984 |
classwords = |
| 985 |
glist_add_ptr(classwords, ckd_salloc(wptr[0])); |
| 986 |
classprobs = glist_add_float32(classprobs, fprob); |
| 987 |
} |
| 988 |
} |
| 989 |
else { |
| 990 |
/* Start a new LM class if the LMCLASS marker is seen */ |
| 991 |
if (n_words == 2 && 0 == strcmp(wptr[0], "LMCLASS")) { |
| 992 |
if (inclass) |
| 993 |
goto error_out; |
| 994 |
inclass = TRUE; |
| 995 |
classname = ckd_salloc(wptr[1]); |
| 996 |
} |
| 997 |
/* Otherwise, just ignore whatever junk we got */ |
| 998 |
} |
| 999 |
} |
| 1000 |
rv = 0; /* Success. */ |
| 1001 |
|
| 1002 |
error_out: |
| 1003 |
/* Free all the stuff we might have allocated. */ |
| 1004 |
fclose_comp(fp, is_pipe); |
| 1005 |
for (gn = classwords; gn; gn = gnode_next(gn)) |
| 1006 |
ckd_free(gnode_ptr(gn)); |
| 1007 |
glist_free(classwords); |
| 1008 |
glist_free(classprobs); |
| 1009 |
ckd_free(classname); |
| 1010 |
|
| 1011 |
return rv; |
| 1012 |
} |
| 1013 |
|
| 1014 |
void |
| 1015 |
classdef_free(classdef_t * classdef) |
| 1016 |
{ |
| 1017 |
int32 i; |
| 1018 |
for (i = 0; i < classdef->n_words; ++i) |
| 1019 |
ckd_free(classdef->words[i]); |
| 1020 |
ckd_free(classdef->words); |
| 1021 |
ckd_free(classdef->weights); |
| 1022 |
ckd_free(classdef); |
| 1023 |
} |
| 1024 |
|
| 1025 |
|
| 1026 |
int32 |
| 1027 |
ngram_model_read_classdef(ngram_model_t * model, const char *file_name) |
| 1028 |
{ |
| 1029 |
hash_table_t *classes; |
| 1030 |
glist_t hl = NULL; |
| 1031 |
gnode_t *gn; |
| 1032 |
int32 rv = -1; |
| 1033 |
|
| 1034 |
classes = hash_table_new(0, FALSE); |
| 1035 |
if (read_classdef_file(classes, file_name) < 0) { |
| 1036 |
hash_table_free(classes); |
| 1037 |
return -1; |
| 1038 |
} |
| 1039 |
|
| 1040 |
/* Create a new class in the language model for each classdef. */ |
| 1041 |
hl = hash_table_tolist(classes, NULL); |
| 1042 |
for (gn = hl; gn; gn = gnode_next(gn)) { |
| 1043 |
hash_entry_t *he = gnode_ptr(gn); |
| 1044 |
classdef_t *classdef = he->val; |
| 1045 |
|
| 1046 |
if (ngram_model_add_class(model, he->key, 1.0, |
| 1047 |
classdef->words, |
| 1048 |
classdef->weights, |
| 1049 |
classdef->n_words) < 0) |
| 1050 |
goto error_out; |
| 1051 |
} |
| 1052 |
rv = 0; |
| 1053 |
|
| 1054 |
error_out: |
| 1055 |
for (gn = hl; gn; gn = gnode_next(gn)) { |
| 1056 |
hash_entry_t *he = gnode_ptr(gn); |
| 1057 |
ckd_free((char *) he->key); |
| 1058 |
classdef_free(he->val); |
| 1059 |
} |
| 1060 |
glist_free(hl); |
| 1061 |
hash_table_free(classes); |
| 1062 |
return rv; |
| 1063 |
} |