SphinxBase  0.6
fsg_model.c
1 /* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /* ====================================================================
3  * Copyright (c) 1999-2004 Carnegie Mellon University. All rights
4  * reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  *
10  * 1. Redistributions of source code must retain the above copyright
11  * notice, this list of conditions and the following disclaimer.
12  *
13  * 2. Redistributions in binary form must reproduce the above copyright
14  * notice, this list of conditions and the following disclaimer in
15  * the documentation and/or other materials provided with the
16  * distribution.
17  *
18  *
19  * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
20  * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
21  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
23  * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30  *
31  * ====================================================================
32  *
33  */
34 
35 /* System headers. */
36 #ifdef _WIN32_WCE
37 /*MC in a debug build it's implicitly included by assert.h
38  but you need this in a release build */
39 #include <windows.h>
40 #else
41 #include <time.h>
42 #endif /* _WIN32_WCE */
43 #include <stdio.h>
44 #include <string.h>
45 #include <assert.h>
46 
47 /* SphinxBase headers. */
48 #include "sphinxbase/err.h"
49 #include "sphinxbase/pio.h"
50 #include "sphinxbase/ckd_alloc.h"
51 #include "sphinxbase/prim_type.h"
52 #include "sphinxbase/strfuncs.h"
53 #include "sphinxbase/hash_table.h"
54 #include "sphinxbase/fsg_model.h"
55 
63 struct trans_list_s {
64  hash_table_t *null_trans; /* Null transitions keyed by state. */
65  hash_table_t *trans; /* Lists of non-null transitions keyed by state. */
66 };
67 
71 struct fsg_arciter_s {
72  hash_iter_t *itor, *null_itor;
73  gnode_t *gn;
74 };
75 
76 #define FSG_MODEL_BEGIN_DECL "FSG_BEGIN"
77 #define FSG_MODEL_END_DECL "FSG_END"
78 #define FSG_MODEL_N_DECL "N"
79 #define FSG_MODEL_NUM_STATES_DECL "NUM_STATES"
80 #define FSG_MODEL_S_DECL "S"
81 #define FSG_MODEL_START_STATE_DECL "START_STATE"
82 #define FSG_MODEL_F_DECL "F"
83 #define FSG_MODEL_FINAL_STATE_DECL "FINAL_STATE"
84 #define FSG_MODEL_T_DECL "T"
85 #define FSG_MODEL_TRANSITION_DECL "TRANSITION"
86 #define FSG_MODEL_COMMENT_CHAR '#'
87 
88 
89 static int32
90 nextline_str2words(FILE * fp, int32 * lineno,
91  char **lineptr, char ***wordptr)
92 {
93  for (;;) {
94  size_t len;
95  int32 n;
96 
97  ckd_free(*lineptr);
98  if ((*lineptr = fread_line(fp, &len)) == NULL)
99  return -1;
100 
101  (*lineno)++;
102 
103  if ((*lineptr)[0] == FSG_MODEL_COMMENT_CHAR)
104  continue; /* Skip comment lines */
105 
106  n = str2words(*lineptr, NULL, 0);
107  if (n == 0)
108  continue; /* Skip blank lines */
109 
110  /* Abuse of realloc(), but this doesn't have to be fast. */
111  if (*wordptr == NULL)
112  *wordptr = ckd_calloc(n, sizeof(**wordptr));
113  else
114  *wordptr = ckd_realloc(*wordptr, n * sizeof(**wordptr));
115  return str2words(*lineptr, *wordptr, n);
116  }
117 }
118 
119 void
120 fsg_model_trans_add(fsg_model_t * fsg,
121  int32 from, int32 to, int32 logp, int32 wid)
122 {
123  fsg_link_t *link;
124  glist_t gl;
125  gnode_t *gn;
126 
127  if (fsg->trans[from].trans == NULL)
128  fsg->trans[from].trans = hash_table_new(5, HASH_CASE_YES);
129 
130  /* Check for duplicate link (i.e., link already exists with label=wid) */
131  for (gn = gl = fsg_model_trans(fsg, from, to); gn; gn = gnode_next(gn)) {
132  link = (fsg_link_t *) gnode_ptr(gn);
133  if (link->wid == wid) {
134  if (link->logs2prob < logp)
135  link->logs2prob = logp;
136  return;
137  }
138  }
139 
140  /* Create transition object */
141  link = listelem_malloc(fsg->link_alloc);
142  link->from_state = from;
143  link->to_state = to;
144  link->logs2prob = logp;
145  link->wid = wid;
146 
147  /* Add it to the list of transitions and update the hash table */
148  gl = glist_add_ptr(gl, (void *)link);
149  hash_table_replace_bkey(fsg->trans[from].trans,
150  (char const *)&link->to_state,
151  sizeof(link->to_state), gl);
152 }
153 
154 int32
155 fsg_model_tag_trans_add(fsg_model_t * fsg, int32 from, int32 to, int32 logp, int32 wid)
156 {
157  fsg_link_t *link, *link2;
158 
159  /* Check for transition probability */
160  if (logp > 0) {
161  E_FATAL("Null transition prob must be <= 1.0 (state %d -> %d)\n",
162  from, to);
163  }
164 
165  /* Self-loop null transitions (with prob <= 1.0) are redundant */
166  if (from == to)
167  return -1;
168 
169  if (fsg->trans[from].null_trans == NULL)
170  fsg->trans[from].null_trans = hash_table_new(5, HASH_CASE_YES);
171 
172  /* Check for a duplicate link; if found, keep the higher prob */
173  link = fsg_model_null_trans(fsg, from, to);
174  if (link) {
175  if (link->logs2prob < logp) {
176  link->logs2prob = logp;
177  return 0;
178  }
179  else
180  return -1;
181  }
182 
183  /* Create null transition object */
184  link = listelem_malloc(fsg->link_alloc);
185  link->from_state = from;
186  link->to_state = to;
187  link->logs2prob = logp;
188  link->wid = -1;
189 
190  link2 = (fsg_link_t *)
191  hash_table_enter_bkey(fsg->trans[from].null_trans,
192  (char const *)&link->to_state,
193  sizeof(link->to_state), link);
194  assert(link == link2);
195 
196  return 1;
197 }
198 
199 int32
200 fsg_model_null_trans_add(fsg_model_t * fsg, int32 from, int32 to, int32 logp)
201 {
202  return fsg_model_tag_trans_add(fsg, from, to, logp, -1);
203 }
204 
205 glist_t
206 fsg_model_null_trans_closure(fsg_model_t * fsg, glist_t nulls)
207 {
208  gnode_t *gn1, *gn2;
209  int updated;
210  fsg_link_t *tl1, *tl2;
211  int32 k, n;
212 
213  E_INFO("Computing transitive closure for null transitions\n");
214 
215  if (nulls == NULL) {
216  fsg_link_t *null;
217  int i, j;
218 
219  for (i = 0; i < fsg->n_state; ++i) {
220  for (j = 0; j < fsg->n_state; ++j) {
221  if ((null = fsg_model_null_trans(fsg, i, j)))
222  nulls = glist_add_ptr(nulls, null);
223  }
224  }
225  }
226 
227  /*
228  * Probably not the most efficient closure implementation, in general, but
229  * probably reasonably efficient for a sparse null transition matrix.
230  */
231  n = 0;
232  do {
233  updated = FALSE;
234 
235  for (gn1 = nulls; gn1; gn1 = gnode_next(gn1)) {
236  tl1 = (fsg_link_t *) gnode_ptr(gn1);
237  assert(tl1->wid < 0);
238 
239  for (gn2 = nulls; gn2; gn2 = gnode_next(gn2)) {
240  tl2 = (fsg_link_t *) gnode_ptr(gn2);
241 
242  if (tl1->to_state == tl2->from_state) {
243  k = fsg_model_null_trans_add(fsg,
244  tl1->from_state,
245  tl2->to_state,
246  tl1->logs2prob +
247  tl2->logs2prob);
248  if (k >= 0) {
249  updated = TRUE;
250  if (k > 0) {
251  nulls =
252  glist_add_ptr(nulls, (void *)
253  fsg_model_null_trans
254  (fsg, tl1->from_state,
255  tl2->to_state));
256  n++;
257  }
258  }
259  }
260  }
261  }
262  } while (updated);
263 
264  E_INFO("%d null transitions added\n", n);
265 
266  return nulls;
267 }
268 
269 glist_t
270 fsg_model_trans(fsg_model_t *fsg, int32 i, int32 j)
271 {
272  void *val;
273 
274  if (fsg->trans[i].trans == NULL)
275  return NULL;
276  if (hash_table_lookup_bkey(fsg->trans[i].trans, (char const *)&j,
277  sizeof(j), &val) < 0)
278  return NULL;
279  return (glist_t)val;
280 }
281 
282 fsg_link_t *
283 fsg_model_null_trans(fsg_model_t *fsg, int32 i, int32 j)
284 {
285  void *val;
286 
287  if (fsg->trans[i].null_trans == NULL)
288  return NULL;
289  if (hash_table_lookup_bkey(fsg->trans[i].null_trans, (char const *)&j,
290  sizeof(j), &val) < 0)
291  return NULL;
292  return (fsg_link_t *)val;
293 }
294 
296 fsg_model_arcs(fsg_model_t *fsg, int32 i)
297 {
298  fsg_arciter_t *itor;
299 
300  if (fsg->trans[i].trans == NULL && fsg->trans[i].null_trans == NULL)
301  return NULL;
302  itor = ckd_calloc(1, sizeof(*itor));
303  if (fsg->trans[i].null_trans)
304  itor->null_itor = hash_table_iter(fsg->trans[i].null_trans);
305  if (fsg->trans[i].trans)
306  itor->itor = hash_table_iter(fsg->trans[i].trans);
307  if (itor->itor != NULL)
308  itor->gn = hash_entry_val(itor->itor->ent);
309  return itor;
310 }
311 
312 fsg_link_t *
313 fsg_arciter_get(fsg_arciter_t *itor)
314 {
315  /* Iterate over non-null arcs first. */
316  if (itor->gn)
317  return (fsg_link_t *)gnode_ptr(itor->gn);
318  else if (itor->null_itor)
319  return (fsg_link_t *)hash_entry_val(itor->null_itor->ent);
320  else
321  return NULL;
322 }
323 
325 fsg_arciter_next(fsg_arciter_t *itor)
326 {
327  /* Iterate over non-null arcs first. */
328  if (itor->gn) {
329  itor->gn = gnode_next(itor->gn);
330  /* Move to the next destination arc. */
331  if (itor->gn == NULL) {
332  itor->itor = hash_table_iter_next(itor->itor);
333  if (itor->itor != NULL)
334  itor->gn = hash_entry_val(itor->itor->ent);
335  else if (itor->null_itor == NULL)
336  goto stop_iteration;
337  }
338  }
339  else {
340  if (itor->null_itor == NULL)
341  goto stop_iteration;
342  itor->null_itor = hash_table_iter_next(itor->null_itor);
343  if (itor->null_itor == NULL)
344  goto stop_iteration;
345  }
346  return itor;
347 stop_iteration:
348  fsg_arciter_free(itor);
349  return NULL;
350 
351 }
352 
353 void
354 fsg_arciter_free(fsg_arciter_t *itor)
355 {
356  if (itor == NULL)
357  return;
358  hash_table_iter_free(itor->null_itor);
359  hash_table_iter_free(itor->itor);
360  ckd_free(itor);
361 }
362 
363 int
364 fsg_model_word_id(fsg_model_t *fsg, char const *word)
365 {
366  int wid;
367 
368  /* Search for an existing word matching this. */
369  for (wid = 0; wid < fsg->n_word; ++wid) {
370  if (0 == strcmp(fsg->vocab[wid], word))
371  break;
372  }
373  /* If not found, add this to the vocab. */
374  if (wid == fsg->n_word)
375  return -1;
376  return wid;
377 }
378 
379 int
380 fsg_model_word_add(fsg_model_t *fsg, char const *word)
381 {
382  int wid;
383 
384  /* Search for an existing word matching this. */
385  wid = fsg_model_word_id(fsg, word);
386  /* If not found, add this to the vocab. */
387  if (wid == -1) {
388  wid = fsg->n_word;
389  if (fsg->n_word == fsg->n_word_alloc) {
390  fsg->n_word_alloc += 10;
391  fsg->vocab = ckd_realloc(fsg->vocab,
392  fsg->n_word_alloc * sizeof(*fsg->vocab));
393  if (fsg->silwords)
394  fsg->silwords = bitvec_realloc(fsg->silwords, fsg->n_word_alloc);
395  if (fsg->altwords)
396  fsg->altwords = bitvec_realloc(fsg->altwords, fsg->n_word_alloc);
397  }
398  ++fsg->n_word;
399  fsg->vocab[wid] = ckd_salloc(word);
400  }
401  return wid;
402 }
403 
404 int
405 fsg_model_add_silence(fsg_model_t * fsg, char const *silword,
406  int state, float32 silprob)
407 {
408  int32 logsilp;
409  int n_trans, silwid, src;
410 
411  E_INFO("Adding silence transitions for %s to FSG\n", silword);
412 
413  silwid = fsg_model_word_add(fsg, silword);
414  logsilp = (int32) (logmath_log(fsg->lmath, silprob) * fsg->lw);
415  if (fsg->silwords == NULL)
416  fsg->silwords = bitvec_alloc(fsg->n_word_alloc);
417  bitvec_set(fsg->silwords, silwid);
418 
419  n_trans = 0;
420  if (state == -1) {
421  for (src = 0; src < fsg->n_state; src++) {
422  fsg_model_trans_add(fsg, src, src, logsilp, silwid);
423  ++n_trans;
424  }
425  }
426  else {
427  fsg_model_trans_add(fsg, state, state, logsilp, silwid);
428  ++n_trans;
429  }
430 
431  E_INFO("Added %d silence word transitions\n", n_trans);
432  return n_trans;
433 }
434 
435 int
436 fsg_model_add_alt(fsg_model_t * fsg, char const *baseword,
437  char const *altword)
438 {
439  int i, basewid, altwid;
440  int ntrans;
441 
442  /* FIXME: This will get slow, eventually... */
443  for (basewid = 0; basewid < fsg->n_word; ++basewid)
444  if (0 == strcmp(fsg->vocab[basewid], baseword))
445  break;
446  if (basewid == fsg->n_word) {
447  E_ERROR("Base word %s not present in FSG vocabulary!\n", baseword);
448  return -1;
449  }
450  altwid = fsg_model_word_add(fsg, altword);
451  if (fsg->altwords == NULL)
452  fsg->altwords = bitvec_alloc(fsg->n_word_alloc);
453  bitvec_set(fsg->altwords, altwid);
454 
455  E_DEBUG(2,("Adding alternate word transitions (%s,%s) to FSG\n",
456  baseword, altword));
457 
458  /* Look for all transitions involving baseword and duplicate them. */
459  /* FIXME: This will also get slow, eventually... */
460  ntrans = 0;
461  for (i = 0; i < fsg->n_state; ++i) {
462  hash_iter_t *itor;
463  if (fsg->trans[i].trans == NULL)
464  continue;
465  for (itor = hash_table_iter(fsg->trans[i].trans); itor;
466  itor = hash_table_iter_next(itor)) {
467  glist_t trans;
468  gnode_t *gn;
469 
470  trans = hash_entry_val(itor->ent);
471  for (gn = trans; gn; gn = gnode_next(gn)) {
472  fsg_link_t *fl = gnode_ptr(gn);
473  if (fl->wid == basewid) {
474  fsg_link_t *link;
475 
476  /* Create transition object */
477  link = listelem_malloc(fsg->link_alloc);
478  link->from_state = fl->from_state;
479  link->to_state = fl->to_state;
480  link->logs2prob = fl->logs2prob; /* FIXME!!!??? */
481  link->wid = altwid;
482 
483  trans =
484  glist_add_ptr(trans, (void *) link);
485  ++ntrans;
486  }
487  }
488  hash_entry_val(itor->ent) = trans;
489  }
490  }
491 
492  E_DEBUG(2,("Added %d alternate word transitions\n", ntrans));
493  return ntrans;
494 }
495 
496 
497 fsg_model_t *
498 fsg_model_init(char const *name, logmath_t *lmath, float32 lw, int32 n_state)
499 {
500  fsg_model_t *fsg;
501 
502  /* Allocate basic stuff. */
503  fsg = ckd_calloc(1, sizeof(*fsg));
504  fsg->refcount = 1;
505  fsg->link_alloc = listelem_alloc_init(sizeof(fsg_link_t));
506  fsg->lmath = lmath;
507  fsg->name = name ? ckd_salloc(name) : NULL;
508  fsg->n_state = n_state;
509  fsg->lw = lw;
510 
511  fsg->trans = ckd_calloc(fsg->n_state, sizeof(*fsg->trans));
512 
513  return fsg;
514 }
515 
516 fsg_model_t *
517 fsg_model_read(FILE * fp, logmath_t *lmath, float32 lw)
518 {
519  fsg_model_t *fsg;
520  hash_table_t *vocab;
521  hash_iter_t *itor;
522  int32 lastwid;
523  char **wordptr;
524  char *lineptr;
525  char *fsgname;
526  int32 lineno;
527  int32 n, i, j;
528  int n_state, n_trans, n_null_trans;
529  glist_t nulls;
530  float32 p;
531 
532  lineno = 0;
533  vocab = hash_table_new(32, FALSE);
534  wordptr = NULL;
535  lineptr = NULL;
536  nulls = NULL;
537  fsgname = NULL;
538  fsg = NULL;
539 
540  /* Scan upto FSG_BEGIN header */
541  for (;;) {
542  n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
543  if (n < 0) {
544  E_ERROR("%s declaration missing\n", FSG_MODEL_BEGIN_DECL);
545  goto parse_error;
546  }
547 
548  if ((strcmp(wordptr[0], FSG_MODEL_BEGIN_DECL) == 0)) {
549  if (n > 2) {
550  E_ERROR("Line[%d]: malformed FSG_BEGIN declaration\n",
551  lineno);
552  goto parse_error;
553  }
554  break;
555  }
556  }
557  /* Save FSG name, or it will get clobbered below :(.
558  * If name is missing, try the default.
559  */
560  if (n == 2) {
561  fsgname = ckd_salloc(wordptr[1]);
562  } else {
563  E_WARN ("FSG name is missing\n");
564  fsgname = ckd_salloc("unknown");
565  }
566 
567  /* Read #states */
568  n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
569  if ((n != 2)
570  || ((strcmp(wordptr[0], FSG_MODEL_N_DECL) != 0)
571  && (strcmp(wordptr[0], FSG_MODEL_NUM_STATES_DECL) != 0))
572  || (sscanf(wordptr[1], "%d", &n_state) != 1)
573  || (n_state <= 0)) {
574  E_ERROR
575  ("Line[%d]: #states declaration line missing or malformed\n",
576  lineno);
577  goto parse_error;
578  }
579 
580  /* Now create the FSG. */
581  fsg = fsg_model_init(fsgname, lmath, lw, n_state);
582  ckd_free(fsgname);
583  fsgname = NULL;
584 
585  /* Read start state */
586  n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
587  if ((n != 2)
588  || ((strcmp(wordptr[0], FSG_MODEL_S_DECL) != 0)
589  && (strcmp(wordptr[0], FSG_MODEL_START_STATE_DECL) != 0))
590  || (sscanf(wordptr[1], "%d", &(fsg->start_state)) != 1)
591  || (fsg->start_state < 0)
592  || (fsg->start_state >= fsg->n_state)) {
593  E_ERROR
594  ("Line[%d]: start state declaration line missing or malformed\n",
595  lineno);
596  goto parse_error;
597  }
598 
599  /* Read final state */
600  n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
601  if ((n != 2)
602  || ((strcmp(wordptr[0], FSG_MODEL_F_DECL) != 0)
603  && (strcmp(wordptr[0], FSG_MODEL_FINAL_STATE_DECL) != 0))
604  || (sscanf(wordptr[1], "%d", &(fsg->final_state)) != 1)
605  || (fsg->final_state < 0)
606  || (fsg->final_state >= fsg->n_state)) {
607  E_ERROR
608  ("Line[%d]: final state declaration line missing or malformed\n",
609  lineno);
610  goto parse_error;
611  }
612 
613  /* Read transitions */
614  lastwid = 0;
615  n_trans = n_null_trans = 0;
616  for (;;) {
617  int32 wid, tprob;
618 
619  n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
620  if (n <= 0) {
621  E_ERROR("Line[%d]: transition or FSG_END statement expected\n",
622  lineno);
623  goto parse_error;
624  }
625 
626  if ((strcmp(wordptr[0], FSG_MODEL_END_DECL) == 0)) {
627  break;
628  }
629 
630  if ((strcmp(wordptr[0], FSG_MODEL_T_DECL) == 0)
631  || (strcmp(wordptr[0], FSG_MODEL_TRANSITION_DECL) == 0)) {
632 
633 
634  if (((n != 4) && (n != 5))
635  || (sscanf(wordptr[1], "%d", &i) != 1)
636  || (sscanf(wordptr[2], "%d", &j) != 1)
637  || (i < 0) || (i >= fsg->n_state)
638  || (j < 0) || (j >= fsg->n_state)) {
639  E_ERROR
640  ("Line[%d]: transition spec malformed; Expecting: from-state to-state trans-prob [word]\n",
641  lineno);
642  goto parse_error;
643  }
644 
645  p = atof_c(wordptr[3]);
646  if ((p <= 0.0) || (p > 1.0)) {
647  E_ERROR
648  ("Line[%d]: transition spec malformed; Expecting float as transition probability\n",
649  lineno);
650  goto parse_error;
651  }
652  }
653  else {
654  E_ERROR("Line[%d]: transition or FSG_END statement expected\n",
655  lineno);
656  goto parse_error;
657  }
658 
659  tprob = (int32)(logmath_log(lmath, p) * fsg->lw);
660  /* Add word to "dictionary". */
661  if (n > 4) {
662  if (hash_table_lookup_int32(vocab, wordptr[4], &wid) < 0) {
663  (void)hash_table_enter_int32(vocab, ckd_salloc(wordptr[4]), lastwid);
664  wid = lastwid;
665  ++lastwid;
666  }
667  fsg_model_trans_add(fsg, i, j, tprob, wid);
668  ++n_trans;
669  }
670  else {
671  if (fsg_model_null_trans_add(fsg, i, j, tprob) == 1) {
672  ++n_null_trans;
673  nulls = glist_add_ptr(nulls, fsg_model_null_trans(fsg, i, j));
674  }
675  }
676  }
677 
678  E_INFO("FSG: %d states, %d unique words, %d transitions (%d null)\n",
679  fsg->n_state, hash_table_inuse(vocab), n_trans, n_null_trans);
680 
681  /* Do transitive closure on null transitions */
682  nulls = fsg_model_null_trans_closure(fsg, nulls);
683  glist_free(nulls);
684 
685  /* Now create a string table from the "dictionary" */
686  fsg->n_word = hash_table_inuse(vocab);
687  fsg->n_word_alloc = fsg->n_word + 10; /* Pad it a bit. */
688  fsg->vocab = ckd_calloc(fsg->n_word_alloc, sizeof(*fsg->vocab));
689  for (itor = hash_table_iter(vocab); itor; itor = hash_table_iter_next(itor)) {
690  char const *word = hash_entry_key(itor->ent);
691  int32 wid = (int32)(long)hash_entry_val(itor->ent);
692  fsg->vocab[wid] = (char *)word;
693  }
694  hash_table_free(vocab);
695  ckd_free(lineptr);
696  ckd_free(wordptr);
697 
698  return fsg;
699 
700  parse_error:
701  for (itor = hash_table_iter(vocab); itor; itor = hash_table_iter_next(itor))
702  ckd_free((char *)hash_entry_key(itor->ent));
703  glist_free(nulls);
704  hash_table_free(vocab);
705  ckd_free(fsgname);
706  ckd_free(lineptr);
707  ckd_free(wordptr);
708  fsg_model_free(fsg);
709  return NULL;
710 }
711 
712 
713 fsg_model_t *
714 fsg_model_readfile(const char *file, logmath_t *lmath, float32 lw)
715 {
716  FILE *fp;
717  fsg_model_t *fsg;
718 
719  if ((fp = fopen(file, "r")) == NULL) {
720  E_ERROR("Failed to open FSG file '%s' for reading: %s\n", file, strerror(errno));
721  return NULL;
722  }
723  fsg = fsg_model_read(fp, lmath, lw);
724  fclose(fp);
725  return fsg;
726 }
727 
728 fsg_model_t *
729 fsg_model_retain(fsg_model_t *fsg)
730 {
731  ++fsg->refcount;
732  return fsg;
733 }
734 
735 static void
736 trans_list_free(fsg_model_t *fsg, int32 i)
737 {
738  hash_iter_t *itor;
739 
740  /* FIXME (maybe): FSG links will all get freed when we call
741  * listelem_alloc_free() so don't bother freeing them explicitly
742  * here. */
743  if (fsg->trans[i].trans) {
744  for (itor = hash_table_iter(fsg->trans[i].trans);
745  itor; itor = hash_table_iter_next(itor)) {
746  glist_t gl = (glist_t)hash_entry_val(itor->ent);
747  glist_free(gl);
748  }
749  }
750  hash_table_free(fsg->trans[i].trans);
751  hash_table_free(fsg->trans[i].null_trans);
752 }
753 
754 int
755 fsg_model_free(fsg_model_t * fsg)
756 {
757  int i;
758 
759  if (fsg == NULL)
760  return 0;
761 
762  if (--fsg->refcount > 0)
763  return fsg->refcount;
764 
765  for (i = 0; i < fsg->n_word; ++i)
766  ckd_free(fsg->vocab[i]);
767  for (i = 0; i < fsg->n_state; ++i)
768  trans_list_free(fsg, i);
769  ckd_free(fsg->trans);
770  ckd_free(fsg->vocab);
772  bitvec_free(fsg->silwords);
773  bitvec_free(fsg->altwords);
774  ckd_free(fsg->name);
775  ckd_free(fsg);
776  return 0;
777 }
778 
779 
780 void
781 fsg_model_write(fsg_model_t * fsg, FILE * fp)
782 {
783  int32 i;
784 
785  fprintf(fp, "%s %s\n", FSG_MODEL_BEGIN_DECL, fsg->name ? fsg->name : "");
786  fprintf(fp, "%s %d\n", FSG_MODEL_NUM_STATES_DECL, fsg->n_state);
787  fprintf(fp, "%s %d\n", FSG_MODEL_START_STATE_DECL, fsg->start_state);
788  fprintf(fp, "%s %d\n", FSG_MODEL_FINAL_STATE_DECL, fsg->final_state);
789 
790  for (i = 0; i < fsg->n_state; i++) {
791  fsg_arciter_t *itor;
792 
793  for (itor = fsg_model_arcs(fsg, i); itor; itor = fsg_arciter_next(itor)) {
794  fsg_link_t *tl = fsg_arciter_get(itor);
795 
796  fprintf(fp, "%s %d %d %f %s\n", FSG_MODEL_TRANSITION_DECL,
797  tl->from_state, tl->to_state,
798  logmath_exp(fsg->lmath, (int32)(tl->logs2prob / fsg->lw)),
799  (tl->wid < 0) ? "" : fsg_model_word_str(fsg, tl->wid));
800  }
801  }
802 
803  fprintf(fp, "%s\n", FSG_MODEL_END_DECL);
804 
805  fflush(fp);
806 }
807 
808 void
809 fsg_model_writefile(fsg_model_t *fsg, char const *file)
810 {
811  FILE *fp;
812 
813  assert(fsg);
814 
815  E_INFO("Writing FSG file '%s'\n", file);
816 
817  if ((fp = fopen(file, "w")) == NULL) {
818  E_ERROR("Failed to open FSG file '%s' for reading: %s\n", file, strerror(errno));
819  return;
820  }
821 
822  fsg_model_write(fsg, fp);
823 
824  fclose(fp);
825 }
826 
827 static void
828 fsg_model_write_fsm_trans(fsg_model_t *fsg, int i, FILE *fp)
829 {
830  fsg_arciter_t *itor;
831 
832  for (itor = fsg_model_arcs(fsg, i); itor;
833  itor = fsg_arciter_next(itor)) {
834  fsg_link_t *tl = fsg_arciter_get(itor);
835  fprintf(fp, "%d %d %s %f\n",
836  tl->from_state, tl->to_state,
837  (tl->wid < 0) ? "<eps>" : fsg_model_word_str(fsg, tl->wid),
838  -logmath_log_to_ln(fsg->lmath, tl->logs2prob / fsg->lw));
839  }
840 }
841 
842 void
843 fsg_model_write_fsm(fsg_model_t * fsg, FILE * fp)
844 {
845  int i;
846 
847  /* Write transitions from initial state first. */
848  fsg_model_write_fsm_trans(fsg, fsg_model_start_state(fsg), fp);
849 
850  /* Other states. */
851  for (i = 0; i < fsg->n_state; i++) {
852  if (i == fsg_model_start_state(fsg))
853  continue;
854  fsg_model_write_fsm_trans(fsg, i, fp);
855  }
856 
857  /* Final state. */
858  fprintf(fp, "%d 0\n", fsg_model_final_state(fsg));
859 
860  fflush(fp);
861 }
862 
863 void
864 fsg_model_writefile_fsm(fsg_model_t *fsg, char const *file)
865 {
866  FILE *fp;
867 
868  assert(fsg);
869 
870  E_INFO("Writing FSM file '%s'\n", file);
871 
872  if ((fp = fopen(file, "w")) == NULL) {
873  E_ERROR("Failed to open fsm file '%s' for writing: %s\n", file, strerror(errno));
874  return;
875  }
876 
877  fsg_model_write_fsm(fsg, fp);
878 
879  fclose(fp);
880 }
881 
882 void
883 fsg_model_write_symtab(fsg_model_t *fsg, FILE *file)
884 {
885  int i;
886 
887  fprintf(file, "<eps> 0\n");
888  for (i = 0; i < fsg_model_n_word(fsg); ++i) {
889  fprintf(file, "%s %d\n", fsg_model_word_str(fsg, i), i + 1);
890  }
891  fflush(file);
892 }
893 
894 void
895 fsg_model_writefile_symtab(fsg_model_t *fsg, char const *file)
896 {
897  FILE *fp;
898 
899  assert(fsg);
900 
901  E_INFO("Writing FSM symbol table '%s'\n", file);
902 
903  if ((fp = fopen(file, "w")) == NULL) {
904  E_ERROR("Failed to open symbol table '%s' for writing: %s\n", file, strerror(errno));
905  return;
906  }
907 
908  fsg_model_write_symtab(fsg, fp);
909 
910  fclose(fp);
911 }