| 1 |
/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ |
| 2 |
/* ==================================================================== |
| 3 |
* Copyright (c) 2015 Carnegie Mellon University. All rights |
| 4 |
* reserved. |
| 5 |
* |
| 6 |
* Redistribution and use in source and binary forms, with or without |
| 7 |
* modification, are permitted provided that the following conditions |
| 8 |
* are met: |
| 9 |
* |
| 10 |
* 1. Redistributions of source code must retain the above copyright |
| 11 |
* notice, this list of conditions and the following disclaimer. |
| 12 |
* |
| 13 |
* 2. Redistributions in binary form must reproduce the above copyright |
| 14 |
* notice, this list of conditions and the following disclaimer in |
| 15 |
* the documentation and/or other materials provided with the |
| 16 |
* distribution. |
| 17 |
* |
| 18 |
* This work was supported in part by funding from the Defense Advanced |
| 19 |
* Research Projects Agency and the National Science Foundation of the |
| 20 |
* United States of America, and the CMU Sphinx Speech Consortium. |
| 21 |
* |
| 22 |
* THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND |
| 23 |
* ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, |
| 24 |
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
| 25 |
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY |
| 26 |
* NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 27 |
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 28 |
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 29 |
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 30 |
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 31 |
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 32 |
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 33 |
* |
| 34 |
* ==================================================================== |
| 35 |
* |
| 36 |
*/ |
| 37 |
|
| 38 |
#ifndef __NGRAMS_RAW_H__ |
| 39 |
#define __NGRAMS_RAW_H__ |
| 40 |
|
| 41 |
#include <sphinxbase/hash_table.h> |
| 42 |
#include <sphinxbase/logmath.h> |
| 43 |
#include <sphinxbase/prim_type.h> |
| 44 |
#include <sphinxbase/pio.h> |
| 45 |
#include <sphinxbase/err.h> |
| 46 |
|
| 47 |
typedef struct ngram_raw_s { |
| 48 |
uint32 *words; /* array of word indexes, length corresponds to ngram order */ |
| 49 |
float32 prob; |
| 50 |
float32 backoff; |
| 51 |
uint32 order; |
| 52 |
} ngram_raw_t; |
| 53 |
|
| 54 |
typedef union { |
| 55 |
float32 f; |
| 56 |
int32 l; |
| 57 |
} dmp_weight_t; |
| 58 |
|
| 59 |
/** |
| 60 |
* Raw ordered ngrams comparator |
| 61 |
*/ |
| 62 |
int ngram_ord_comparator(const void *a_raw, const void *b_raw); |
| 63 |
|
| 64 |
/** |
| 65 |
* Read ngrams of order > 1 from ARPA file |
| 66 |
* @param li [in] sphinxbase file line iterator that point to bigram description in ARPA file |
| 67 |
* @param wid [in] hashtable that maps string word representation to id |
| 68 |
* @param lmath [in] log math used for log convertions |
| 69 |
* @param counts [in] amount of ngrams for each order |
| 70 |
* @param order [in] maximum order of ngrams |
| 71 |
* @return raw ngrams of order bigger than 1 |
| 72 |
*/ |
| 73 |
ngram_raw_t **ngrams_raw_read_arpa(lineiter_t ** li, logmath_t * lmath, |
| 74 |
uint32 * counts, int order, |
| 75 |
hash_table_t * wid); |
| 76 |
|
| 77 |
/** |
| 78 |
* Reads ngrams of order > 1 from DMP file. |
| 79 |
* @param fp [in] file to read from. Position in file corresponds to start of bigram description |
| 80 |
* @param lmath [in] log math used for log convertions |
| 81 |
* @param counts [in] amount of ngrams for each order |
| 82 |
* @param order [in] maximum order of ngrams |
| 83 |
* @param unigram_next [in] array of next word pointers for unigrams. Needed to define forst word of bigrams |
| 84 |
* @param do_swap [in] wether to do swap of bits |
| 85 |
* @return raw ngrams of order bigger than 1 |
| 86 |
*/ |
| 87 |
ngram_raw_t **ngrams_raw_read_dmp(FILE * fp, logmath_t * lmath, |
| 88 |
uint32 * counts, int order, |
| 89 |
uint32 * unigram_next, uint8 do_swap); |
| 90 |
|
| 91 |
void ngrams_raw_free(ngram_raw_t ** raw_ngrams, uint32 * counts, |
| 92 |
int order); |
| 93 |
|
| 94 |
#endif /* __LM_NGRAMS_RAW_H__ */ |