55 #include <sphinxbase/prim_type.h>
56 #include <sphinxbase/ckd_alloc.h>
57 #include <sphinxbase/byteorder.h>
58 #include <sphinxbase/case.h>
59 #include <sphinxbase/err.h>
66 bin_mdef_read_text(cmd_ln_t *
config,
const char *filename)
70 int i, nodes, ci_idx, lc_idx, rc_idx;
73 if ((mdef =
mdef_init((
char *) filename, TRUE)) == NULL)
78 E_ERROR(
"Number of senones exceeds limit: %d > %d\n",
84 E_ERROR(
"Number of senone sequences exceeds limit: %d > %d\n",
91 bmdef = ckd_calloc(1,
sizeof(*bmdef));
118 bmdef->
ciname[0] = ckd_calloc(nchars, 1);
122 bmdef->
ciname[i - 1] + strlen(bmdef->
ciname[i - 1]) + 1;
124 if (i > 0 && strcmp(bmdef->
ciname[i - 1], bmdef->
ciname[i]) > 0) {
126 E_ERROR(
"Phone names are not in sorted order, sorry.");
127 bin_mdef_free(bmdef);
134 for (i = 0; i < mdef->
n_phone; ++i) {
137 if (i < bmdef->n_ciphone) {
142 bmdef->
phone[i].info.cd.ctx[0] = mdef->
phone[i].ci;
143 bmdef->
phone[i].info.cd.ctx[1] = mdef->
phone[i].lc;
150 nodes = lc_idx = ci_idx = rc_idx = 0;
158 for (rc = lc->rclist; rc; rc = rc->next) {
173 E_INFO(
"Allocating %d * %d bytes (%d KiB) for CD tree\n",
174 nodes,
sizeof(*bmdef->
cd_tree),
175 nodes *
sizeof(*bmdef->
cd_tree) / 1024);
185 E_INFO(
"%d => %c (%d@%d)\n",
201 for (rc = lc->rclist; rc; rc = rc->next) {
206 E_INFO(
"%d => %s %s %s %c (%d@%d)\n",
225 E_INFO(
"%d => %s %s %c (%d@%d)\n",
242 E_INFO(
"%d => %d=%s (%d@%d)\n",
243 ci_idx, j, bmdef->
ciname[j],
274 case BIN_MDEF_FROM_TEXT:
276 ckd_free(m->
sseq[0]);
280 case BIN_MDEF_IN_MEMORY:
283 case BIN_MDEF_ON_DISK:
296 static const char format_desc[] =
297 "BEGIN FILE FORMAT DESCRIPTION\n"
298 "int32 n_ciphone; /**< Number of base (CI) phones */\n"
299 "int32 n_phone; /**< Number of base (CI) phones + (CD) triphones */\n"
300 "int32 n_emit_state; /**< Number of emitting states per phone (0 if heterogeneous) */\n"
301 "int32 n_ci_sen; /**< Number of CI senones; these are the first */\n"
302 "int32 n_sen; /**< Number of senones (CI+CD) */\n"
303 "int32 n_tmat; /**< Number of transition matrices */\n"
304 "int32 n_sseq; /**< Number of unique senone sequences */\n"
305 "int32 n_ctx; /**< Number of phones of context */\n"
306 "int32 n_cd_tree; /**< Number of nodes in CD tree structure */\n"
307 "int32 sil; /**< CI phone ID for silence */\n"
308 "char ciphones[][]; /**< CI phone strings (null-terminated) */\n"
309 "char padding[]; /**< Padding to a 4-bytes boundary */\n"
310 "struct { int16 ctx; int16 n_down; int32 pid/down } cd_tree[];\n"
311 "struct { int32 ssid; int32 tmat; int8 attr[4] } phones[];\n"
312 "int16 sseq[]; /**< Unique senone sequences */\n"
313 "int8 sseq_len[]; /**< Number of states in each sseq (none if homogeneous) */\n"
314 "END FILE FORMAT DESCRIPTION\n";
317 bin_mdef_read(cmd_ln_t *
config,
const char *filename)
322 int32 val, i, swap, pos, end;
327 if ((m = bin_mdef_read_text(config, filename)) != NULL)
330 E_INFO(
"Reading binary model definition: %s\n", filename);
331 if ((fh = fopen(filename,
"rb")) == NULL)
334 if (fread(&val, 4, 1, fh) != 1) {
336 E_ERROR_SYSTEM(
"Failed to read byte-order marker from %s\n",
341 if (val == BIN_MDEF_OTHER_ENDIAN) {
343 E_INFO(
"Must byte-swap %s\n", filename);
345 if (fread(&val, 4, 1, fh) != 1) {
347 E_ERROR_SYSTEM(
"Failed to read version from %s\n", filename);
352 if (val > BIN_MDEF_FORMAT_VERSION) {
353 E_ERROR(
"File format version %d for %s is newer than library\n",
358 if (fread(&val, 4, 1, fh) != 1) {
360 E_ERROR_SYSTEM(
"Failed to read header length from %s\n", filename);
366 fseek(fh, val, SEEK_CUR);
369 m = ckd_calloc(1,
sizeof(*m));
373 #define FREAD_SWAP32_CHK(dest) \
374 if (fread((dest), 4, 1, fh) != 1) { \
377 E_ERROR_SYSTEM("Failed to read %s from %s\n", #dest, filename); \
380 if (swap) SWAP_INT32(dest);
386 FREAD_SWAP32_CHK(&m->
n_sen);
387 FREAD_SWAP32_CHK(&m->
n_tmat);
388 FREAD_SWAP32_CHK(&m->
n_sseq);
389 FREAD_SWAP32_CHK(&m->
n_ctx);
391 FREAD_SWAP32_CHK(&m->
sil);
397 do_mmap = config ? cmd_ln_boolean_r(config,
"-mmap") : TRUE;
399 E_WARN(
"-mmap specified, but mdef is other-endian. Will not memory-map.\n");
404 m->
filemap = mmio_file_read(filename);
418 fseek(fh, 0, SEEK_END);
420 fseek(fh, pos, SEEK_SET);
421 m->
ciname[0] = ckd_malloc(end - pos);
422 if (fread(m->
ciname[0], 1, end - pos, fh) != end - pos)
423 E_FATAL(
"Failed to read %d bytes of data from %s\n", end - pos, filename);
432 tree_start = (tree_start + 3) & ~3;
443 for (i = 0; i < m->
n_phone; ++i) {
450 SWAP_INT32(sseq_size);
452 m->
sseq[0] = (uint16 *) (sseq_size + 1);
454 for (i = 0; i < *sseq_size; ++i)
455 SWAP_INT16(m->
sseq[0] + i);
458 for (i = 1; i < m->
n_sseq; ++i)
463 for (i = 1; i < m->
n_sseq; ++i)
477 for (; i < m->
n_sen; ++i)
479 for (i = 0; i < m->
n_sen; ++i)
481 for (i = 0; i < m->
n_phone; ++i) {
484 for (j = 0; j < bin_mdef_n_emit_state_phone(m, i); ++j) {
485 int s = bin_mdef_sseq2sen(m, ssid, j);
486 int ci = bin_mdef_pid2ci(m, i);
492 (
"Senone %d is shared between multiple base phones\n",
495 if (j > bin_mdef_n_emit_state_phone(m, ci))
496 E_WARN(
"CD phone %d has fewer states than CI phone %d\n",
500 bin_mdef_sseq2sen(m, m->
phone[ci].
ssid, j);
508 (
"%d CI-phone, %d CD-phone, %d emitstate/phone, %d CI-sen, %d Sen, %d Sen-Seq\n",
521 if ((fh = fopen(filename,
"wb")) == NULL)
525 val = BIN_MDEF_NATIVE_ENDIAN;
526 fwrite(&val, 1, 4, fh);
528 val = BIN_MDEF_FORMAT_VERSION;
529 fwrite(&val, 1,
sizeof(val), fh);
532 val = ((
sizeof(format_desc) + 3) & ~3);
533 fwrite(&val, 1,
sizeof(val), fh);
534 fwrite(format_desc, 1,
sizeof(format_desc), fh);
537 fwrite(&i, 1, val -
sizeof(format_desc), fh);
544 fwrite(&m->
n_sen, 4, 1, fh);
545 fwrite(&m->
n_tmat, 4, 1, fh);
546 fwrite(&m->
n_sseq, 4, 1, fh);
547 fwrite(&m->
n_ctx, 4, 1, fh);
554 fwrite(&val, 4, 1, fh);
560 val = (ftell(fh) + 3) & ~3;
562 fwrite(&i, 1, val - ftell(fh), fh);
571 fwrite(&val, 4, 1, fh);
574 fwrite(m->
sseq[0],
sizeof(**m->
sseq),
582 for (i = 0; i < m->
n_sseq; ++i)
586 fwrite(&n, 4, 1, fh);
589 fwrite(m->
sseq[0],
sizeof(**m->
sseq), n, fh);
603 int p, i, n_total_state;
605 if (strcmp(filename,
"-") == 0)
608 if ((fh = fopen(filename,
"w")) == NULL)
612 fprintf(fh,
"0.3\n");
613 fprintf(fh,
"%d n_base\n", m->
n_ciphone);
619 for (i = 0; i < m->
n_phone; ++i)
622 fprintf(fh,
"%d n_state_map\n", n_total_state);
623 fprintf(fh,
"%d n_tied_state\n", m->
n_sen);
624 fprintf(fh,
"%d n_tied_ci_state\n", m->
n_ci_sen);
625 fprintf(fh,
"%d n_tied_tmat\n", m->
n_tmat);
626 fprintf(fh,
"#\n# Columns definitions\n");
627 fprintf(fh,
"#%4s %3s %3s %1s %6s %4s %s\n",
628 "base",
"lft",
"rt",
"p",
"attrib",
"tmat",
629 " ... state id's ...");
634 fprintf(fh,
"%5s %3s %3s %1s", m->
ciname[p],
"-",
"-",
"-");
636 if (bin_mdef_is_fillerphone(m, p))
637 fprintf(fh,
" %6s",
"filler");
639 fprintf(fh,
" %6s",
"n/a");
646 for (i = 0; i < n_state; i++) {
656 fprintf(fh,
"%5s %3s %3s %c",
662 if (bin_mdef_is_fillerphone(m, p))
663 fprintf(fh,
" %6s",
"filler");
665 fprintf(fh,
" %6s",
"n/a");
673 for (i = 0; i < n_state; i++) {
679 if (strcmp(filename,
"-") != 0)
695 mid = (low + high) / 2;
696 c = strcmp(ciphone, m->
ciname[mid]);
708 bin_mdef_ciphone_id_nocase(
bin_mdef_t * m,
const char *ciphone)
718 mid = (low + high) / 2;
719 c = strcmp_nocase(ciphone, m->
ciname[mid]);
734 assert(ci < m->n_ciphone);
739 bin_mdef_phone_id(
bin_mdef_t * m, int32 ci, int32 lc, int32 rc, int32 wpos)
749 if (lc < 0 || rc < 0)
752 assert((ci >= 0) && (ci < m->n_ciphone));
753 assert((lc >= 0) && (lc < m->n_ciphone));
754 assert((rc >= 0) && (rc < m->n_ciphone));
760 ctx[2] = (m->
sil >= 0
761 && m->
phone[lc].info.
ci.filler) ? m->
sil : lc;
762 ctx[3] = (m->
sil >= 0
763 && m->
phone[rc].info.
ci.filler) ? m->
sil : rc;
773 E_INFO(
"Looking for context %d=%s in %d at %d\n",
774 ctx[level], m->
ciname[ctx[level]],
777 for (i = 0; i < max; ++i) {
779 E_INFO(
"Look at context %d=%s at %d\n",
783 if (cd_tree[i].ctx == ctx[level])
789 E_INFO(
"Found context %d=%s at %d, n_down=%d, down=%d\n",
790 ctx[level], m->
ciname[ctx[level]],
795 if (cd_tree[i].n_down == 0)
796 return cd_tree[i].c.
pid;
808 bin_mdef_phone_id_nearest(
bin_mdef_t * m, int32 b, int32 l, int32 r, int32 pos)
819 p = bin_mdef_phone_id(m, b, l, r, pos);
826 p = bin_mdef_phone_id(m, b, l, r, tmppos);
835 int newl = l, newr = r;
836 if (m->
phone[(
int)l].info.
ci.filler
839 if (m->
phone[(
int)r].info.
ci.filler
842 if ((newl != l) || (newr != r)) {
843 p = bin_mdef_phone_id(m, b, newl, newr, pos);
849 p = bin_mdef_phone_id(m, b, newl, newr, tmppos);
867 assert((pid >= 0) && (pid < m->n_phone));
871 if (pid < m->n_ciphone)
872 sprintf(buf,
"%s", bin_mdef_ciphone_str(m, pid));
874 sprintf(buf,
"%s %s %s %c",
875 bin_mdef_ciphone_str(m, m->
phone[pid].info.cd.ctx[0]),
876 bin_mdef_ciphone_str(m, m->
phone[pid].info.cd.ctx[1]),
877 bin_mdef_ciphone_str(m, m->
phone[pid].info.cd.ctx[2]),
878 wpos_name[m->
phone[pid].info.cd.wpos]);