Debian Codegraph @

src/libsphinxbase/lm/ngram_model_internal.h

raw
1 /* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /* ====================================================================
3 * Copyright (c) 1999-2007 Carnegie Mellon University. All rights
4 * reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 *
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 *
18 * This work was supported in part by funding from the Defense Advanced
19 * Research Projects Agency and the National Science Foundation of the
20 * United States of America, and the CMU Sphinx Speech Consortium.
21 *
22 * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
23 * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
24 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
26 * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 *
34 * ====================================================================
35 *
36 */
37 /*
38 * \file ngram_model_internal.h Internal structures for N-Gram models
39 *
40 * Author: David Huggins-Daines <[email protected]>
41 */
42
43 #ifndef __NGRAM_MODEL_INTERNAL_H__
44 #define __NGRAM_MODEL_INTERNAL_H__
45
46 #include "sphinxbase/ngram_model.h"
47 #include "sphinxbase/hash_table.h"
48
49 /**
50 * Common implementation of ngram_model_t.
51 *
52 * The details of bigram, trigram, and higher-order N-gram storage, if any, can
53 * vary somewhat depending on the file format in use.
54 */
55 struct ngram_model_s {
56 int refcount; /**< Reference count */
57 uint32 *n_counts; /**< Counts for 1, 2, 3, ... grams */
58 int32 n_1g_alloc; /**< Number of allocated word strings (for new word addition) */
59 int32 n_words; /**< Number of actual word strings (NOT the same as the
60 number of unigrams, due to class words). */
61
62 uint8 n; /**< This is an n-gram model (1, 2, 3, ...). */
63 uint8 n_classes; /**< Number of classes (maximum 128) */
64 uint8 writable; /**< Are word strings writable? */
65 uint8 flags; /**< Any other flags we might care about
66 (FIXME: Merge this and writable) */
67 logmath_t *lmath; /**< Log-math object */
68 float32 lw; /**< Language model scaling factor */
69 int32 log_wip; /**< Log of word insertion penalty */
70 int32 log_zero; /**< Zero probability, cached here for quick lookup */
71 char **word_str; /**< Unigram names */
72 hash_table_t *wid; /**< Mapping of unigram names to word IDs. */
73 int32 *tmp_wids; /**< Temporary array of word IDs for ngram_model_get_ngram() */
74 struct ngram_class_s **classes; /**< Word class definitions. */
75 struct ngram_funcs_s *funcs; /**< Implementation-specific methods. */
76 };
77
78 /**
79 * Implementation of ngram_class_t.
80 */
81 struct ngram_class_s {
82 int32 tag_wid; /**< Base word ID for this class tag */
83 int32 start_wid; /**< Starting base word ID for this class' words */
84 int32 n_words; /**< Number of base words for this class */
85 int32 *prob1; /**< Probability table for base words */
86 /**
87 * Custom hash table for additional words.
88 */
89 struct ngram_hash_s {
90 int32 wid; /**< Word ID of this bucket */
91 int32 prob1; /**< Probability for this word */
92 int32 next; /**< Index of next bucket (or -1 for no collision) */
93 } *nword_hash;
94 int32 n_hash; /**< Number of buckets in nword_hash (power of 2) */
95 int32 n_hash_inuse; /**< Number of words in nword_hash */
96 };
97
98 #define NGRAM_MAX_ORDER 5
99
100 #define NGRAM_HASH_SIZE 128
101
102 #define NGRAM_BASEWID(wid) ((wid)&0xffffff)
103 #define NGRAM_CLASSID(wid) (((wid)>>24) & 0x7f)
104 #define NGRAM_CLASSWID(wid,classid) (((classid)<<24) | 0x80000000 | (wid))
105 #define NGRAM_IS_CLASSWID(wid) ((wid)&0x80000000)
106
107 #define UG_ALLOC_STEP 10
108
109 /** Implementation-specific functions for operating on ngram_model_t objects */
110 typedef struct ngram_funcs_s {
111 /**
112 * Implementation-specific function for freeing an ngram_model_t.
113 */
114 void (*free) (ngram_model_t * model);
115 /**
116 * Implementation-specific function for applying language model weights.
117 */
118 int (*apply_weights) (ngram_model_t * model, float32 lw, float32 wip);
119 /**
120 * Implementation-specific function for querying language model score.
121 */
122 int32(*score) (ngram_model_t * model,
123 int32 wid,
124 int32 * history, int32 n_hist, int32 * n_used);
125 /**
126 * Implementation-specific function for querying raw language
127 * model probability.
128 */
129 int32(*raw_score) (ngram_model_t * model,
130 int32 wid,
131 int32 * history, int32 n_hist, int32 * n_used);
132 /**
133 * Implementation-specific function for adding unigrams.
134 *
135 * This function updates the internal structures of a language
136 * model to add the given unigram with the given weight (defined
137 * as a log-factor applied to the uniform distribution). This
138 * includes reallocating or otherwise resizing the set of unigrams.
139 *
140 * @return The language model score (not raw log-probability) of
141 * the new word, or 0 for failure.
142 */
143 int32(*add_ug) (ngram_model_t * model, int32 wid, int32 lweight);
144
145 /**
146 * Implementation-specific function for purging N-Gram cache
147 */
148 void (*flush) (ngram_model_t * model);
149 } ngram_funcs_t;
150
151 /**
152 * One class definition from a classdef file.
153 */
154 typedef struct classdef_s {
155 char **words;
156 float32 *weights;
157 int32 n_words;
158 } classdef_t;
159
160 /**
161 * Initialize the base ngram_model_t structure.
162 */
163 int32
164 ngram_model_init(ngram_model_t * model,
165 ngram_funcs_t * funcs,
166 logmath_t * lmath, int32 n, int32 n_unigram);
167
168 /**
169 * Read a probdef file.
170 */
171 int32 read_classdef_file(hash_table_t * classes,
172 const char *classdef_file);
173
174 /**
175 * Free a class definition.
176 */
177 void classdef_free(classdef_t * classdef);
178
179 /**
180 * Allocate and initialize an N-Gram class.
181 */
182 ngram_class_t *ngram_class_new(ngram_model_t * model, int32 tag_wid,
183 int32 start_wid, glist_t classwords);
184
185 /**
186 * Deallocate an N-Gram class.
187 */
188 void ngram_class_free(ngram_class_t * lmclass);
189
190 /**
191 * Get the in-class log probability for a word in an N-Gram class.
192 *
193 * @return This probability, or 1 if word not found.
194 */
195 int32 ngram_class_prob(ngram_class_t * lmclass, int32 wid);
196
197 #endif /* __NGRAM_MODEL_INTERNAL_H__ */