eSpeak NG is an open source speech synthesizer that supports more than hundred languages and accents.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

compiledict.cpp 41KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975
  1. /***************************************************************************
  2. * Copyright (C) 2005 to 2013 by Jonathan Duddington *
  3. * email: [email protected] *
  4. * *
  5. * This program is free software; you can redistribute it and/or modify *
  6. * it under the terms of the GNU General Public License as published by *
  7. * the Free Software Foundation; either version 3 of the License, or *
  8. * (at your option) any later version. *
  9. * *
  10. * This program is distributed in the hope that it will be useful, *
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of *
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
  13. * GNU General Public License for more details. *
  14. * *
  15. * You should have received a copy of the GNU General Public License *
  16. * along with this program; if not, write see: *
  17. * <http://www.gnu.org/licenses/>. *
  18. ***************************************************************************/
  19. #include "StdAfx.h"
  20. #include <stdio.h>
  21. #include <ctype.h>
  22. #include <stdlib.h>
  23. #include <string.h>
  24. #include <wctype.h>
  25. #include "speak_lib.h"
  26. #include "speech.h"
  27. #include "phoneme.h"
  28. #include "synthesize.h"
  29. #include "translate.h"
  30. extern void Write4Bytes(FILE *f, int value);
  31. int HashDictionary(const char *string);
  32. static FILE *f_log = NULL;
  33. extern char *dir_dictionary;
  34. extern char word_phonemes[N_WORD_PHONEMES]; // a word translated into phoneme codes
  35. static int linenum;
  36. static int error_count;
  37. static int text_mode = 0;
  38. static int debug_flag = 0;
  39. static int error_need_dictionary = 0;
  40. static int hash_counts[N_HASH_DICT];
  41. static char *hash_chains[N_HASH_DICT];
  42. static char letterGroupsDefined[N_LETTER_GROUPS];
  43. MNEM_TAB mnem_rules[] = {
  44. {"unpr", 0x01},
  45. {"w_alt1", 0x11},
  46. {"w_alt2", 0x12},
  47. {"w_alt3", 0x13},
  48. {"w_alt4", 0x14},
  49. {"w_alt5", 0x15},
  50. {"w_alt6", 0x16},
  51. {"w_alt", 0x11}, // note: put longer names before their sub-strings
  52. {"p_alt1", 0x21},
  53. {"p_alt2", 0x22},
  54. {"p_alt3", 0x23},
  55. {"p_alt4", 0x24},
  56. {"p_alt5", 0x25},
  57. {"p_alt6", 0x26},
  58. {"p_alt", 0x21},
  59. {NULL, -1} };
  60. MNEM_TAB mnem_flags[] = {
  61. // these in the first group put a value in bits0-3 of dictionary_flags
  62. {"$1", 0x41}, // stress on 1st syllable
  63. {"$2", 0x42}, // stress on 2nd syllable
  64. {"$3", 0x43},
  65. {"$4", 0x44},
  66. {"$5", 0x45},
  67. {"$6", 0x46},
  68. {"$7", 0x47},
  69. {"$u", 0x48}, // reduce to unstressed
  70. {"$u1", 0x49},
  71. {"$u2", 0x4a},
  72. {"$u3", 0x4b},
  73. {"$u+", 0x4c}, // reduce to unstressed, but stress at end of clause
  74. {"$u1+", 0x4d},
  75. {"$u2+", 0x4e},
  76. {"$u3+", 0x4f},
  77. // these set the corresponding numbered bit if dictionary_flags
  78. {"$pause", 8}, // ensure pause before this word
  79. {"$strend", 9}, // full stress if at end of clause
  80. {"$strend2", 10}, // full stress if at end of clause, or only followed by unstressed
  81. {"$unstressend",11}, // reduce stress at end of clause
  82. {"$abbrev", 13}, // use this pronuciation rather than split into letters
  83. // language specific
  84. {"$double", 14}, // IT double the initial consonant of next word
  85. {"$alt", 15}, // use alternative pronunciation
  86. {"$alt1", 15}, // synonym for $alt
  87. {"$alt2", 16},
  88. {"$alt3", 17},
  89. {"$alt4", 18},
  90. {"$alt5", 19},
  91. {"$alt6", 20},
  92. {"$combine", 23}, // Combine with the next word
  93. {"$dot", 24}, // ignore '.' after this word (abbreviation)
  94. {"$hasdot", 25}, // use this pronunciation if there is a dot after the word
  95. {"$max3", 27}, // limit to 3 repetitions
  96. {"$brk", 28}, // a shorter $pause
  97. {"$text", 29}, // word translates to replcement text, not phonemes
  98. // flags in dictionary word 2
  99. {"$verbf", 0x20}, // verb follows
  100. {"$verbsf", 0x21}, // verb follows, allow -s suffix
  101. {"$nounf", 0x22}, // noun follows
  102. {"$pastf", 0x23}, // past tense follows
  103. {"$verb", 0x24}, // use this pronunciation when its a verb
  104. {"$noun", 0x25}, // use this pronunciation when its a noun
  105. {"$past", 0x26}, // use this pronunciation when its past tense
  106. {"$verbextend",0x28}, // extend influence of 'verb follows'
  107. {"$capital", 0x29}, // use this pronunciation if initial letter is upper case
  108. {"$allcaps", 0x2a}, // use this pronunciation if initial letter is upper case
  109. {"$accent", 0x2b}, // character name is base-character name + accent name
  110. {"$sentence",0x2d}, // only if this clause is a sentence (i.e. terminator is {. ? !} not {, ; :}
  111. {"$only", 0x2e}, // only match on this word without suffix
  112. {"$onlys", 0x2f}, // only match with none, or with 's' suffix
  113. {"$stem", 0x30}, // must have a suffix
  114. {"$atend", 0x31}, // use this pronunciation if at end of clause
  115. {"$atstart", 0x32}, // use this pronunciation at start of clause
  116. // doesn't set dictionary_flags
  117. {"$?", 100}, // conditional rule, followed by byte giving the condition number
  118. {"$textmode", 200},
  119. {"$phonememode", 201},
  120. {NULL, -1}
  121. };
  122. #define LEN_GROUP_NAME 12
  123. typedef struct {
  124. char name[LEN_GROUP_NAME+1];
  125. unsigned int start;
  126. unsigned int length;
  127. int group3_ix;
  128. } RGROUP;
  129. int isspace2(unsigned int c)
  130. {//=========================
  131. // can't use isspace() because on Windows, isspace(0xe1) gives TRUE !
  132. int c2;
  133. if(((c2 = (c & 0xff)) == 0) || (c > ' '))
  134. return(0);
  135. return(1);
  136. }
  137. static FILE *fopen_log(const char *fname,const char *access)
  138. {//==================================================
  139. // performs fopen, but produces error message to f_log if it fails
  140. FILE *f;
  141. if((f = fopen(fname,access)) == NULL)
  142. {
  143. if(f_log != NULL)
  144. fprintf(f_log,"Can't access (%s) file '%s'\n",access,fname);
  145. }
  146. return(f);
  147. }
  148. const char *LookupMnemName(MNEM_TAB *table, const int value)
  149. //==========================================================
  150. /* Lookup a mnemonic string in a table, return its name */
  151. {
  152. while(table->mnem != NULL)
  153. {
  154. if(table->value==value)
  155. return(table->mnem);
  156. table++;
  157. }
  158. return(""); /* not found */
  159. } /* end of LookupMnemValue */
  160. void print_dictionary_flags(unsigned int *flags, char *buf, int buf_len)
  161. {//========================================================================
  162. int stress;
  163. int ix;
  164. const char *name;
  165. int len;
  166. int total = 0;
  167. buf[0] = 0;
  168. if((stress = flags[0] & 0xf) != 0)
  169. {
  170. sprintf(buf, "%s", LookupMnemName(mnem_flags, stress + 0x40));
  171. total = strlen(buf);
  172. buf += total;
  173. }
  174. for(ix=8; ix<64; ix++)
  175. {
  176. if(((ix < 30) && (flags[0] & (1 << ix))) || ((ix >= 0x20) && (flags[1] & (1 << (ix-0x20)))))
  177. {
  178. name = LookupMnemName(mnem_flags, ix);
  179. len = strlen(name) + 1;
  180. total += len;
  181. if(total >= buf_len)
  182. continue;
  183. sprintf(buf, " %s", name);
  184. buf += len;
  185. }
  186. }
  187. }
  188. char *DecodeRule(const char *group_chars, int group_length, char *rule, int control)
  189. {//=================================================================================
  190. /* Convert compiled match template to ascii */
  191. unsigned char rb;
  192. unsigned char c;
  193. char *p;
  194. int ix;
  195. int match_type;
  196. int finished=0;
  197. int value;
  198. int linenum=0;
  199. int flags;
  200. int suffix_char;
  201. int condition_num=0;
  202. int at_start = 0;
  203. const char *name;
  204. char buf[60];
  205. char buf_pre[60];
  206. char suffix[20];
  207. static char output[60];
  208. static char symbols[] = {' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
  209. '&','%','+','#','S','D','Z','A','L','!',' ','@','?','J','N','K','V','?','T','X','?','W'};
  210. static char symbols_lg[] = {'A','B','C','H','F','G','Y'};
  211. match_type = 0;
  212. buf_pre[0] = 0;
  213. for(ix=0; ix<group_length; ix++)
  214. {
  215. buf[ix] = group_chars[ix];
  216. }
  217. buf[ix] = 0;
  218. p = &buf[strlen(buf)];
  219. while(!finished)
  220. {
  221. rb = *rule++;
  222. if(rb <= RULE_LINENUM)
  223. {
  224. switch(rb)
  225. {
  226. case 0:
  227. case RULE_PHONEMES:
  228. finished=1;
  229. break;
  230. case RULE_PRE_ATSTART:
  231. at_start = 1; // drop through to next case
  232. case RULE_PRE:
  233. match_type = RULE_PRE;
  234. *p = 0;
  235. p = buf_pre;
  236. break;
  237. case RULE_POST:
  238. match_type = RULE_POST;
  239. *p = 0;
  240. strcat(buf," (");
  241. p = &buf[strlen(buf)];
  242. break;
  243. case RULE_PH_COMMON:
  244. break;
  245. case RULE_CONDITION:
  246. /* conditional rule, next byte gives condition number */
  247. condition_num = *rule++;
  248. break;
  249. case RULE_LINENUM:
  250. value = (rule[1] & 0xff) - 1;
  251. linenum = (rule[0] & 0xff) - 1 + (value * 255);
  252. rule+=2;
  253. break;
  254. }
  255. continue;
  256. }
  257. if(rb == RULE_DOLLAR)
  258. {
  259. value = *rule++ & 0xff;
  260. if((value != 0x01) || (control & FLAG_UNPRON_TEST))
  261. {
  262. p[0] = '$';
  263. name = LookupMnemName(mnem_rules, value);
  264. strcpy(&p[1],name);
  265. p += (strlen(name)+1);
  266. }
  267. c = ' ';
  268. }
  269. else
  270. if(rb == RULE_ENDING)
  271. {
  272. static const char *flag_chars = "eipvdfq tba ";
  273. flags = ((rule[0] & 0x7f)<< 8) + (rule[1] & 0x7f);
  274. suffix_char = 'S';
  275. if(flags & (SUFX_P >> 8))
  276. suffix_char = 'P';
  277. sprintf(suffix,"%c%d",suffix_char,rule[2] & 0x7f);
  278. rule += 3;
  279. for(ix=0;ix<9;ix++)
  280. {
  281. if(flags & 1)
  282. sprintf(&suffix[strlen(suffix)],"%c",flag_chars[ix]);
  283. flags = (flags >> 1);
  284. }
  285. strcpy(p,suffix);
  286. p += strlen(suffix);
  287. c = ' ';
  288. }
  289. else
  290. if(rb == RULE_LETTERGP)
  291. {
  292. c = symbols_lg[*rule++ - 'A'];
  293. }
  294. else
  295. if(rb == RULE_LETTERGP2)
  296. {
  297. value = *rule++ - 'A';
  298. p[0] = 'L';
  299. p[1] = (value / 10) + '0';
  300. c = (value % 10) + '0';
  301. if(match_type == RULE_PRE)
  302. {
  303. p[0] = c;
  304. c = 'L';
  305. }
  306. p+=2;
  307. }
  308. else
  309. if(rb <= RULE_LAST_RULE)
  310. c = symbols[rb];
  311. else
  312. if(rb == RULE_SPACE)
  313. c = '_';
  314. else
  315. c = rb;
  316. *p++ = c;
  317. }
  318. *p = 0;
  319. p = output;
  320. if(linenum > 0)
  321. {
  322. sprintf(p,"%5d:\t",linenum);
  323. p += 7;
  324. }
  325. if(condition_num > 0)
  326. {
  327. sprintf(p,"?%d ",condition_num);
  328. p = &p[strlen(p)];
  329. }
  330. if(((ix = strlen(buf_pre)) > 0) || at_start)
  331. {
  332. if(at_start)
  333. *p++ = '_';
  334. while(--ix >= 0)
  335. *p++ = buf_pre[ix];
  336. *p++ = ')';
  337. *p++ = ' ';
  338. }
  339. *p = 0;
  340. strcat(p,buf);
  341. ix = strlen(output);
  342. while(ix < 8)
  343. output[ix++]=' ';
  344. output[ix]=0;
  345. return(output);
  346. } /* end of DecodeRule */
  347. static int compile_line(char *linebuf, char *dict_line, int *hash)
  348. {//===============================================================
  349. // Compile a line in the language_list file
  350. unsigned char c;
  351. char *p;
  352. char *word;
  353. char *phonetic;
  354. unsigned int ix;
  355. int step;
  356. unsigned int n_flag_codes = 0;
  357. int flagnum;
  358. int flag_offset;
  359. int length;
  360. int multiple_words = 0;
  361. int multiple_numeric_hyphen = 0;
  362. char *multiple_string = NULL;
  363. char *multiple_string_end = NULL;
  364. int len_word;
  365. int len_phonetic;
  366. int text_not_phonemes; // this word specifies replacement text, not phonemes
  367. unsigned int wc;
  368. int all_upper_case;
  369. char *mnemptr;
  370. unsigned char flag_codes[100];
  371. char encoded_ph[200];
  372. unsigned char bad_phoneme[4];
  373. static char nullstring[] = {0};
  374. text_not_phonemes = 0;
  375. phonetic = word = nullstring;
  376. p = linebuf;
  377. // while(isspace2(*p)) p++;
  378. #ifdef deleted
  379. if(*p == '$')
  380. {
  381. if(memcmp(p,"$textmode",9) == 0)
  382. {
  383. text_mode = 1;
  384. return(0);
  385. }
  386. if(memcmp(p,"$phonememode",12) == 0)
  387. {
  388. text_mode = 0;
  389. return(0);
  390. }
  391. }
  392. #endif
  393. step = 0;
  394. c = 0;
  395. while(c != '\n')
  396. {
  397. c = *p;
  398. if((c == '?') && (step==0))
  399. {
  400. // conditional rule, allow only if the numbered condition is set for the voice
  401. flag_offset = 100;
  402. p++;
  403. if(*p == '!')
  404. {
  405. // allow only if the numbered condition is NOT set
  406. flag_offset = 132;
  407. p++;
  408. }
  409. ix = 0;
  410. if(isdigit(*p))
  411. {
  412. ix += (*p-'0');
  413. p++;
  414. }
  415. if(isdigit(*p))
  416. {
  417. ix = ix*10 + (*p-'0');
  418. p++;
  419. }
  420. flag_codes[n_flag_codes++] = ix + flag_offset;
  421. c = *p;
  422. }
  423. if((c == '$') && isalnum(p[1]))
  424. {
  425. /* read keyword parameter */
  426. mnemptr = p;
  427. while(!isspace2(c = *p)) p++;
  428. *p = 0;
  429. flagnum = LookupMnem(mnem_flags,mnemptr);
  430. if(flagnum > 0)
  431. {
  432. if(flagnum == 200)
  433. {
  434. text_mode = 1;
  435. }
  436. else
  437. if(flagnum == 201)
  438. {
  439. text_mode = 0;
  440. }
  441. else
  442. if(flagnum == BITNUM_FLAG_TEXTMODE)
  443. {
  444. text_not_phonemes = 1;
  445. }
  446. else
  447. {
  448. flag_codes[n_flag_codes++] = flagnum;
  449. }
  450. }
  451. else
  452. {
  453. fprintf(f_log,"%5d: Unknown keyword: %s\n",linenum,mnemptr);
  454. error_count++;
  455. }
  456. }
  457. if((c == '/') && (p[1] == '/') && (multiple_words==0))
  458. {
  459. c = '\n'; /* "//" treat comment as end of line */
  460. }
  461. switch(step)
  462. {
  463. case 0:
  464. if(c == '(')
  465. {
  466. multiple_words = 1;
  467. word = p+1;
  468. step = 1;
  469. }
  470. else
  471. if(!isspace2(c))
  472. {
  473. word = p;
  474. step = 1;
  475. }
  476. break;
  477. case 1:
  478. if((c == '-') && multiple_words)
  479. {
  480. if(isdigit(word[0]))
  481. {
  482. multiple_numeric_hyphen = 1;
  483. }
  484. // else // ???
  485. {
  486. flag_codes[n_flag_codes++] = BITNUM_FLAG_HYPHENATED;
  487. }
  488. c = ' ';
  489. }
  490. if(isspace2(c))
  491. {
  492. p[0] = 0; /* terminate english word */
  493. if(multiple_words)
  494. {
  495. multiple_string = multiple_string_end = p+1;
  496. step = 2;
  497. }
  498. else
  499. {
  500. step = 3;
  501. }
  502. }
  503. else
  504. if(c == ')')
  505. {
  506. if(multiple_words)
  507. {
  508. p[0] = 0;
  509. multiple_words = 0;
  510. step = 3;
  511. }
  512. else
  513. if(word[0] != '_')
  514. {
  515. fprintf(f_log, "%5d: Missing '('\n", linenum);
  516. error_count++;
  517. step = 3;
  518. }
  519. }
  520. break;
  521. case 2:
  522. if(isspace2(c))
  523. {
  524. multiple_words++;
  525. }
  526. else
  527. if(c == ')')
  528. {
  529. p[0] = ' '; // terminate extra string
  530. multiple_string_end = p+1;
  531. step = 3;
  532. }
  533. break;
  534. case 3:
  535. if(!isspace2(c))
  536. {
  537. phonetic = p;
  538. step = 4;
  539. }
  540. break;
  541. case 4:
  542. if(isspace2(c))
  543. {
  544. p[0] = 0; /* terminate phonetic */
  545. step = 5;
  546. }
  547. break;
  548. case 5:
  549. break;
  550. }
  551. p++;
  552. }
  553. if(word[0] == 0)
  554. {
  555. return(0); /* blank line */
  556. }
  557. if(text_mode)
  558. text_not_phonemes = 1;
  559. if(text_not_phonemes)
  560. {
  561. if(word[0] == '_')
  562. {
  563. // This is a special word, used by eSpeak. Translate this into phonemes now
  564. strcat(phonetic, " "); // need a space to indicate word-boundary
  565. // PROBLEM vowel reductions are not applied to the translated phonemes
  566. // condition rules are not applied
  567. TranslateWord(translator,phonetic,0,NULL,NULL);
  568. text_not_phonemes = 0;
  569. strncpy0(encoded_ph, word_phonemes, N_WORD_BYTES-4);
  570. if((word_phonemes[0] == 0) && (error_need_dictionary < 3))
  571. {
  572. // the dictionary was not loaded, we need a second attempt
  573. error_need_dictionary++;
  574. fprintf(f_log,"%5d: Need to compile dictionary again\n",linenum);
  575. }
  576. {
  577. //char decoded_phonemes[128];
  578. //DecodePhonemes(word_phonemes,decoded_phonemes);
  579. //printf("Translator %x %s [%s] [%s]\n",translator->translator_name,word,phonetic,decoded_phonemes);
  580. }
  581. }
  582. else
  583. {
  584. // this is replacement text, so don't encode as phonemes. Restrict the length of the replacement word
  585. strncpy0(encoded_ph,phonetic,N_WORD_BYTES-4);
  586. }
  587. }
  588. else
  589. {
  590. EncodePhonemes(phonetic,encoded_ph,bad_phoneme);
  591. if(strchr(encoded_ph,phonSWITCH) != 0)
  592. {
  593. flag_codes[n_flag_codes++] = BITNUM_FLAG_ONLY_S; // don't match on suffixes (except 's') when switching languages
  594. }
  595. // check for errors in the phonemes codes
  596. if(bad_phoneme[0] != 0)
  597. {
  598. // unrecognised phoneme, report error
  599. fprintf(f_log,"%5d: Bad phoneme [%c] (0x%x) in: %s %s\n",linenum,bad_phoneme[0],bad_phoneme[0],word,phonetic);
  600. error_count++;
  601. }
  602. }
  603. if(text_not_phonemes != translator->langopts.textmode)
  604. {
  605. flag_codes[n_flag_codes++] = BITNUM_FLAG_TEXTMODE;
  606. }
  607. if(sscanf(word,"U+%x",&wc) == 1)
  608. {
  609. // Character code
  610. ix = utf8_out(wc, word);
  611. word[ix] = 0;
  612. }
  613. else
  614. if(word[0] != '_')
  615. {
  616. // convert to lower case, and note if the word is all-capitals
  617. int c2;
  618. all_upper_case = 1;
  619. p = word;
  620. for(p=word;;)
  621. {
  622. // this assumes that the lower case char is the same length as the upper case char
  623. // OK, except for Turkish "I", but use towlower() rather than towlower2()
  624. ix = utf8_in(&c2,p);
  625. if(c2 == 0)
  626. break;
  627. if(iswupper(c2))
  628. {
  629. utf8_out(towlower(c2),p);
  630. }
  631. else
  632. {
  633. all_upper_case = 0;
  634. }
  635. p += ix;
  636. }
  637. if(all_upper_case)
  638. {
  639. flag_codes[n_flag_codes++] = BITNUM_FLAG_ALLCAPS;
  640. }
  641. }
  642. len_word = strlen(word);
  643. if(translator->transpose_min > 0)
  644. {
  645. len_word = TransposeAlphabet(translator, word);
  646. }
  647. *hash = HashDictionary(word);
  648. len_phonetic = strlen(encoded_ph);
  649. dict_line[1] = len_word; // bit 6 indicates whether the word has been compressed
  650. len_word &= 0x3f;
  651. memcpy(&dict_line[2],word,len_word);
  652. if(len_phonetic == 0)
  653. {
  654. // no phonemes specified. set bit 7
  655. dict_line[1] |= 0x80;
  656. length = len_word + 2;
  657. }
  658. else
  659. {
  660. length = len_word + len_phonetic + 3;
  661. strcpy(&dict_line[(len_word)+2],encoded_ph);
  662. }
  663. for(ix=0; ix<n_flag_codes; ix++)
  664. {
  665. dict_line[ix+length] = flag_codes[ix];
  666. }
  667. length += n_flag_codes;
  668. if((multiple_string != NULL) && (multiple_words > 0))
  669. {
  670. if(multiple_words > 10)
  671. {
  672. fprintf(f_log,"%5d: Two many parts in a multi-word entry: %d\n",linenum,multiple_words);
  673. error_count++;
  674. }
  675. else
  676. {
  677. dict_line[length++] = 80 + multiple_words;
  678. ix = multiple_string_end - multiple_string;
  679. if(multiple_numeric_hyphen)
  680. {
  681. dict_line[length++] = ' '; // ???
  682. }
  683. memcpy(&dict_line[length],multiple_string,ix);
  684. length += ix;
  685. }
  686. }
  687. dict_line[0] = length;
  688. return(length);
  689. } /* end of compile_line */
  690. static void compile_dictlist_start(void)
  691. {//=====================================
  692. // initialise dictionary list
  693. int ix;
  694. char *p;
  695. char *p2;
  696. for(ix=0; ix<N_HASH_DICT; ix++)
  697. {
  698. p = hash_chains[ix];
  699. while(p != NULL)
  700. {
  701. memcpy(&p2,p,sizeof(char *));
  702. free(p);
  703. p = p2;
  704. }
  705. hash_chains[ix] = NULL;
  706. hash_counts[ix]=0;
  707. }
  708. }
  709. static void compile_dictlist_end(FILE *f_out)
  710. {//==========================================
  711. // Write out the compiled dictionary list
  712. int hash;
  713. int length;
  714. char *p;
  715. if(f_log != NULL)
  716. {
  717. #ifdef OUTPUT_FORMAT
  718. for(hash=0; hash<N_HASH_DICT; hash++)
  719. {
  720. fprintf(f_log,"%8d",hash_counts[hash]);
  721. if((hash & 7) == 7)
  722. fputc('\n',f_log);
  723. }
  724. fflush(f_log);
  725. #endif
  726. }
  727. for(hash=0; hash<N_HASH_DICT; hash++)
  728. {
  729. p = hash_chains[hash];
  730. hash_counts[hash] = (int)ftell(f_out);
  731. while(p != NULL)
  732. {
  733. length = *(p+sizeof(char *));
  734. fwrite(p+sizeof(char *),length,1,f_out);
  735. memcpy(&p,p,sizeof(char *));
  736. }
  737. fputc(0,f_out);
  738. }
  739. }
  740. static int compile_dictlist_file(const char *path, const char* filename)
  741. {//=====================================================================
  742. int length;
  743. int hash;
  744. char *p;
  745. int count=0;
  746. FILE *f_in;
  747. char buf[200];
  748. char fname[sizeof(path_home)+45];
  749. char dict_line[128];
  750. text_mode = 0;
  751. // try with and without '.txt' extension
  752. sprintf(fname,"%s%s.txt",path,filename);
  753. if((f_in = fopen(fname,"r")) == NULL)
  754. {
  755. sprintf(fname,"%s%s",path,filename);
  756. if((f_in = fopen(fname,"r")) == NULL)
  757. return(-1);
  758. }
  759. fprintf(f_log,"Compiling: '%s'\n",fname);
  760. linenum=0;
  761. while(fgets(buf,sizeof(buf),f_in) != NULL)
  762. {
  763. linenum++;
  764. length = compile_line(buf,dict_line,&hash);
  765. if(length == 0) continue; /* blank line */
  766. hash_counts[hash]++;
  767. p = (char *)malloc(length+sizeof(char *));
  768. if(p == NULL)
  769. {
  770. if(f_log != NULL)
  771. {
  772. fprintf(f_log,"Can't allocate memory\n");
  773. error_count++;
  774. }
  775. break;
  776. }
  777. memcpy(p,&hash_chains[hash],sizeof(char *));
  778. hash_chains[hash] = p;
  779. memcpy(p+sizeof(char *),dict_line,length);
  780. count++;
  781. }
  782. fprintf(f_log,"\t%d entries\n",count);
  783. fclose(f_in);
  784. return(0);
  785. } /* end of compile_dictlist_file */
  786. static char rule_cond[80];
  787. static char rule_pre[80];
  788. static char rule_post[80];
  789. static char rule_match[80];
  790. static char rule_phonemes[80];
  791. static char group_name[LEN_GROUP_NAME+1];
  792. static int group3_ix;
  793. #define N_RULES 2000 // max rules for each group
  794. int isHexDigit(int c)
  795. {
  796. if((c >= '0') && (c <= '9'))
  797. return(c - '0');
  798. if((c >= 'a') && (c <= 'f'))
  799. return(c - 'a' + 10);
  800. if((c >= 'A') && (c <= 'F'))
  801. return(c - 'A' + 10);
  802. return(-1);
  803. }
  804. static void copy_rule_string(char *string, int &state)
  805. {//===================================================
  806. // state 0: conditional, 1=pre, 2=match, 3=post, 4=phonemes
  807. static char *outbuf[5] = {rule_cond, rule_pre, rule_match, rule_post, rule_phonemes};
  808. static int next_state[5] = {2,2,4,4,4};
  809. char *output;
  810. char *p;
  811. int ix;
  812. int len;
  813. char c;
  814. int c2, c3;
  815. int sxflags;
  816. int value;
  817. int literal;
  818. int hexdigit_input = 0;
  819. MNEM_TAB *mr;
  820. if(string[0] == 0) return;
  821. output = outbuf[state];
  822. if(state==4)
  823. {
  824. // append to any previous phoneme string, i.e. allow spaces in the phoneme string
  825. len = strlen(rule_phonemes);
  826. if(len > 0)
  827. rule_phonemes[len++] = ' ';
  828. output = &rule_phonemes[len];
  829. }
  830. sxflags = 0x808000; // to ensure non-zero bytes
  831. for(p=string,ix=0;;)
  832. {
  833. literal = 0;
  834. c = *p++;
  835. if((c == '0') && (p[0] == 'x') && (isHexDigit(p[1]) >= 0) && (isHexDigit(p[2]) >= 0))
  836. {
  837. hexdigit_input = 1;
  838. c = p[1];
  839. p+= 2;
  840. }
  841. if(c == '\\')
  842. {
  843. c = *p++; // treat next character literally
  844. //#ifdef deleted
  845. if((c >= '0') && (c <= '3') && (p[0] >= '0') && (p[0] <= '7') && (p[1] >= '0') && (p[1] <= '7'))
  846. {
  847. // character code given by 3 digit octal value;
  848. c = (c-'0')*64 + (p[0]-'0')*8 + (p[1]-'0');
  849. p += 2;
  850. }
  851. //endif
  852. literal = 1;
  853. }
  854. if(hexdigit_input)
  855. {
  856. if(((c2 = isHexDigit(c)) >= 0) && ((c3 = isHexDigit(p[0])) >= 0))
  857. {
  858. c = c2 * 16 + c3;
  859. literal = 1;
  860. p++;
  861. }
  862. else
  863. {
  864. hexdigit_input = 0;
  865. }
  866. }
  867. if((state==1) || (state==3))
  868. {
  869. // replace special characters (note: 'E' is reserved for a replaced silent 'e')
  870. if(literal == 0)
  871. {
  872. static const char lettergp_letters[9] = {LETTERGP_A,LETTERGP_B,LETTERGP_C,0,0,LETTERGP_F,LETTERGP_G,LETTERGP_H,LETTERGP_Y};
  873. switch(c)
  874. {
  875. case '_':
  876. c = RULE_SPACE;
  877. break;
  878. case 'Y':
  879. c = 'I'; // drop through to next case
  880. case 'A': // vowel
  881. case 'B':
  882. case 'C':
  883. case 'H':
  884. case 'F':
  885. case 'G':
  886. if(state == 1)
  887. {
  888. // pre-rule, put the number before the RULE_LETTERGP;
  889. output[ix++] = lettergp_letters[c-'A'] + 'A';
  890. c = RULE_LETTERGP;
  891. }
  892. else
  893. {
  894. output[ix++] = RULE_LETTERGP;
  895. c = lettergp_letters[c-'A'] + 'A';
  896. }
  897. break;
  898. case 'D':
  899. c = RULE_DIGIT;
  900. break;
  901. case 'K':
  902. c = RULE_NOTVOWEL;
  903. break;
  904. case 'N':
  905. c = RULE_NO_SUFFIX;
  906. break;
  907. case 'V':
  908. c = RULE_IFVERB;
  909. break;
  910. case 'Z':
  911. c = RULE_NONALPHA;
  912. break;
  913. case '+':
  914. c = RULE_INC_SCORE;
  915. break;
  916. case '@':
  917. c = RULE_SYLLABLE;
  918. break;
  919. case '&':
  920. c = RULE_STRESSED;
  921. break;
  922. case '%':
  923. c = RULE_DOUBLE;
  924. break;
  925. case '#':
  926. c = RULE_DEL_FWD;
  927. break;
  928. case '!':
  929. c = RULE_CAPITAL;
  930. break;
  931. case 'T':
  932. output[ix++] = RULE_DOLLAR;
  933. c = 0x11;
  934. break;
  935. case 'W':
  936. c = RULE_SPELLING;
  937. break;
  938. case 'X':
  939. c = RULE_NOVOWELS;
  940. break;
  941. case 'J':
  942. c = RULE_SKIPCHARS;
  943. break;
  944. case 'L':
  945. // expect two digits
  946. c = *p++ - '0';
  947. value = *p++ - '0';
  948. c = c * 10 + value;
  949. if((value < 0) || (value > 9))
  950. {
  951. c = 0;
  952. fprintf(f_log,"%5d: Expected 2 digits after 'L'\n",linenum);
  953. error_count++;
  954. }
  955. else
  956. if((c <= 0) || (c >= N_LETTER_GROUPS) || (letterGroupsDefined[(int)c] == 0))
  957. {
  958. fprintf(f_log,"%5d: Letter group L%.2d not defined\n",linenum,c);
  959. error_count++;
  960. }
  961. c += 'A';
  962. if(state == 1)
  963. {
  964. // pre-rule, put the group number before the RULE_LETTERGP command
  965. output[ix++] = c;
  966. c = RULE_LETTERGP2;
  967. }
  968. else
  969. {
  970. output[ix++] = RULE_LETTERGP2;
  971. }
  972. break;
  973. case '$':
  974. output[ix++] = RULE_DOLLAR;
  975. c = 0;
  976. mr = mnem_rules;
  977. while(mr->mnem != NULL)
  978. {
  979. len = strlen(mr->mnem);
  980. if(memcmp(p, mr->mnem, len) == 0)
  981. {
  982. c = mr->value;
  983. p += len;
  984. break;
  985. }
  986. mr++;
  987. }
  988. if(c == 0)
  989. {
  990. fprintf(f_log,"%5d: $ command not recognized\n",linenum);
  991. error_count++;
  992. }
  993. break;
  994. case 'P':
  995. sxflags |= SUFX_P; // Prefix, now drop through to Suffix
  996. case 'S':
  997. output[ix++] = RULE_ENDING;
  998. value = 0;
  999. while(!isspace2(c = *p++) && (c != 0))
  1000. {
  1001. switch(c)
  1002. {
  1003. case 'e':
  1004. sxflags |= SUFX_E;
  1005. break;
  1006. case 'i':
  1007. sxflags |= SUFX_I;
  1008. break;
  1009. case 'p': // obsolete, replaced by 'P' above
  1010. sxflags |= SUFX_P;
  1011. break;
  1012. case 'v':
  1013. sxflags |= SUFX_V;
  1014. break;
  1015. case 'd':
  1016. sxflags |= SUFX_D;
  1017. break;
  1018. case 'f':
  1019. sxflags |= SUFX_F;
  1020. break;
  1021. case 'q':
  1022. sxflags |= SUFX_Q;
  1023. break;
  1024. case 't':
  1025. sxflags |= SUFX_T;
  1026. break;
  1027. case 'b':
  1028. sxflags |= SUFX_B;
  1029. break;
  1030. case 'a':
  1031. sxflags |= SUFX_A;
  1032. break;
  1033. case 'm':
  1034. sxflags |= SUFX_M;
  1035. break;
  1036. default:
  1037. if(isdigit(c))
  1038. value = (value*10) + (c - '0');
  1039. break;
  1040. }
  1041. }
  1042. p--;
  1043. output[ix++] = sxflags >> 16;
  1044. output[ix++] = sxflags >> 8;
  1045. c = value | 0x80;
  1046. break;
  1047. }
  1048. }
  1049. }
  1050. output[ix++] = c;
  1051. if(c == 0) break;
  1052. }
  1053. state = next_state[state];
  1054. } // end of copy_rule_string
  1055. static char *compile_rule(char *input)
  1056. {//===================================
  1057. int ix;
  1058. unsigned char c;
  1059. int wc;
  1060. char *p;
  1061. char *prule;
  1062. int len;
  1063. int len_name;
  1064. int start;
  1065. int state=2;
  1066. int finish=0;
  1067. char buf[80];
  1068. char output[150];
  1069. unsigned char bad_phoneme[4];
  1070. buf[0]=0;
  1071. rule_cond[0]=0;
  1072. rule_pre[0]=0;
  1073. rule_post[0]=0;
  1074. rule_match[0]=0;
  1075. rule_phonemes[0]=0;
  1076. p = buf;
  1077. for(ix=0; finish==0; ix++)
  1078. {
  1079. c = input[ix];
  1080. switch(c = input[ix])
  1081. {
  1082. case ')': // end of prefix section
  1083. *p = 0;
  1084. state = 1;
  1085. copy_rule_string(buf,state);
  1086. p = buf;
  1087. break;
  1088. case '(': // start of suffix section
  1089. *p = 0;
  1090. state = 2;
  1091. copy_rule_string(buf,state);
  1092. state = 3;
  1093. p = buf;
  1094. if(input[ix+1] == ' ')
  1095. {
  1096. fprintf(f_log,"%5d: Syntax error. Space after (\n",linenum);
  1097. error_count++;
  1098. }
  1099. break;
  1100. case '\n': // end of line
  1101. case '\r':
  1102. case 0: // end of line
  1103. *p = 0;
  1104. copy_rule_string(buf,state);
  1105. finish=1;
  1106. break;
  1107. case '\t': // end of section section
  1108. case ' ':
  1109. *p = 0;
  1110. copy_rule_string(buf,state);
  1111. p = buf;
  1112. break;
  1113. case '?':
  1114. if(state==2)
  1115. state=0;
  1116. else
  1117. *p++ = c;
  1118. break;
  1119. default:
  1120. *p++ = c;
  1121. break;
  1122. }
  1123. }
  1124. if(strcmp(rule_match,"$group")==0)
  1125. strcpy(rule_match,group_name);
  1126. if(rule_match[0]==0)
  1127. {
  1128. if(rule_post[0] != 0)
  1129. {
  1130. fprintf(f_log,"%5d: Syntax error\n",linenum);
  1131. error_count++;
  1132. }
  1133. return(NULL);
  1134. }
  1135. EncodePhonemes(rule_phonemes,buf,bad_phoneme);
  1136. if(bad_phoneme[0] != 0)
  1137. {
  1138. fprintf(f_log,"%5d: Bad phoneme [%c] in %s\n",linenum,bad_phoneme[0],input);
  1139. error_count++;
  1140. }
  1141. strcpy(output,buf);
  1142. len = strlen(buf)+1;
  1143. len_name = strlen(group_name);
  1144. if((len_name > 0) && (memcmp(rule_match,group_name,len_name) != 0))
  1145. {
  1146. utf8_in(&wc,rule_match);
  1147. if((group_name[0] == '9') && IsDigit(wc))
  1148. {
  1149. // numeric group, rule_match starts with a digit, so OK
  1150. }
  1151. else
  1152. {
  1153. fprintf(f_log,"%5d: Wrong initial letters '%s' for group '%s'\n",linenum,rule_match,group_name);
  1154. error_count++;
  1155. }
  1156. }
  1157. strcpy(&output[len],rule_match);
  1158. len += strlen(rule_match);
  1159. if(debug_flag)
  1160. {
  1161. output[len] = RULE_LINENUM;
  1162. output[len+1] = (linenum % 255) + 1;
  1163. output[len+2] = (linenum / 255) + 1;
  1164. len+=3;
  1165. }
  1166. if(rule_cond[0] != 0)
  1167. {
  1168. ix = -1;
  1169. if(rule_cond[0] == '!')
  1170. {
  1171. // allow the rule only if the condition number is NOT set for the voice
  1172. ix = atoi(&rule_cond[1]) + 32;
  1173. }
  1174. else
  1175. {
  1176. // allow the rule only if the condition number is set for the voice
  1177. ix = atoi(rule_cond);
  1178. }
  1179. if((ix > 0) && (ix < 255))
  1180. {
  1181. output[len++] = RULE_CONDITION;
  1182. output[len++] = ix;
  1183. }
  1184. else
  1185. {
  1186. fprintf(f_log,"%5d: bad condition number ?%d\n",linenum,ix);
  1187. error_count++;
  1188. }
  1189. }
  1190. if(rule_pre[0] != 0)
  1191. {
  1192. start = 0;
  1193. if(rule_pre[0] == RULE_SPACE)
  1194. {
  1195. // omit '_' at the beginning of the pre-string and imply it by using RULE_PRE_ATSTART
  1196. c = RULE_PRE_ATSTART;
  1197. start = 1;
  1198. }
  1199. else
  1200. {
  1201. c = RULE_PRE;
  1202. }
  1203. output[len++] = c;
  1204. // output PRE string in reverse order
  1205. for(ix = strlen(rule_pre)-1; ix>=start; ix--)
  1206. output[len++] = rule_pre[ix];
  1207. }
  1208. if(rule_post[0] != 0)
  1209. {
  1210. sprintf(&output[len],"%c%s",RULE_POST,rule_post);
  1211. len += (strlen(rule_post)+1);
  1212. }
  1213. output[len++]=0;
  1214. prule = (char *)malloc(len);
  1215. memcpy(prule,output,len);
  1216. return(prule);
  1217. } // end of compile_rule
  1218. int __cdecl string_sorter(char **a, char **b)
  1219. {//===========================================
  1220. char *pa, *pb;
  1221. int ix;
  1222. if((ix = strcmp(pa = *a,pb = *b)) != 0)
  1223. return(ix);
  1224. pa += (strlen(pa)+1);
  1225. pb += (strlen(pb)+1);
  1226. return(strcmp(pa,pb));
  1227. } /* end of string_sorter */
  1228. static int __cdecl rgroup_sorter(RGROUP *a, RGROUP *b)
  1229. {//===================================================
  1230. // Sort long names before short names
  1231. int ix;
  1232. ix = strlen(b->name) - strlen(a->name);
  1233. if(ix != 0) return(ix);
  1234. ix = strcmp(a->name,b->name);
  1235. if(ix != 0) return(ix);
  1236. return(a->start-b->start);
  1237. }
  1238. #ifdef OUTPUT_FORMAT
  1239. static void print_rule_group(FILE *f_out, int n_rules, char **rules, char *name)
  1240. {//=============================================================================
  1241. int rule;
  1242. int ix;
  1243. unsigned char c;
  1244. int len1;
  1245. int len2;
  1246. int spaces;
  1247. char *p;
  1248. char *pout;
  1249. int condition;
  1250. char buf[80];
  1251. char suffix[12];
  1252. static unsigned char symbols[] = {'@','&','%','+','#','$','D','Z','A','B','C','F'};
  1253. fprintf(f_out,"\n$group %s\n",name);
  1254. for(rule=0; rule<n_rules; rule++)
  1255. {
  1256. p = rules[rule];
  1257. len1 = strlen(p) + 1;
  1258. p = &p[len1];
  1259. len2 = strlen(p);
  1260. rule_match[0]=0;
  1261. rule_pre[0]=0;
  1262. rule_post[0]=0;
  1263. condition = 0;
  1264. pout = rule_match;
  1265. for(ix=0; ix<len2; ix++)
  1266. {
  1267. switch(c = p[ix])
  1268. {
  1269. case RULE_PRE:
  1270. *pout = 0;
  1271. pout = rule_pre;
  1272. break;
  1273. case RULE_POST:
  1274. *pout = 0;
  1275. pout = rule_post;
  1276. break;
  1277. case RULE_CONDITION:
  1278. condition = p[++ix];
  1279. break;
  1280. case RULE_ENDING:
  1281. sprintf(suffix,"$%d[%x]",(p[ix+2]),p[ix+1] & 0x7f);
  1282. ix += 2;
  1283. strcpy(pout,suffix);
  1284. pout += strlen(suffix);
  1285. break;
  1286. default:
  1287. if(c <= RULE_LETTER7)
  1288. c = symbols[c-RULE_SYLLABLE];
  1289. if(c == ' ')
  1290. c = '_';
  1291. *pout++ = c;
  1292. break;
  1293. }
  1294. }
  1295. *pout = 0;
  1296. spaces = 12;
  1297. if(condition > 0)
  1298. {
  1299. sprintf(buf,"?%d ",condition);
  1300. spaces -= strlen(buf);
  1301. fprintf(f_out,"%s",buf);
  1302. }
  1303. if(rule_pre[0] != 0)
  1304. {
  1305. p = buf;
  1306. for(ix=strlen(rule_pre)-1;ix>=0;ix--)
  1307. *p++ = rule_pre[ix];
  1308. sprintf(p,") ");
  1309. spaces -= strlen(buf);
  1310. for(ix=0; ix<spaces; ix++)
  1311. fputc(' ',f_out);
  1312. fprintf(f_out,"%s",buf);
  1313. spaces = 0;
  1314. }
  1315. for(ix=0; ix<spaces; ix++)
  1316. fputc(' ',f_out);
  1317. spaces = 14;
  1318. sprintf(buf," %s ",rule_match);
  1319. if(rule_post[0] != 0)
  1320. {
  1321. p = &buf[strlen(buf)];
  1322. sprintf(p,"(%s ",rule_post);
  1323. }
  1324. fprintf(f_out,"%s",buf);
  1325. spaces -= strlen(buf);
  1326. for(ix=0; ix<spaces; ix++)
  1327. fputc(' ',f_out);
  1328. DecodePhonemes(rules[rule],buf);
  1329. fprintf(f_out,"%s\n",buf); // phonemes
  1330. }
  1331. }
  1332. #endif
  1333. //#define LIST_GROUP_INFO
  1334. static void output_rule_group(FILE *f_out, int n_rules, char **rules, char *name)
  1335. {//==============================================================================
  1336. int ix;
  1337. int len1;
  1338. int len2;
  1339. int len_name;
  1340. char *p;
  1341. char *p2, *p3;
  1342. const char *common;
  1343. short nextchar_count[256];
  1344. memset(nextchar_count,0,sizeof(nextchar_count));
  1345. len_name = strlen(name);
  1346. #ifdef OUTPUT_FORMAT
  1347. print_rule_group(f_log,n_rules,rules,name);
  1348. #endif
  1349. // sort the rules in this group by their phoneme string
  1350. common = "";
  1351. qsort((void *)rules,n_rules,sizeof(char *),(int (__cdecl *)(const void *,const void *))string_sorter);
  1352. if(strcmp(name,"9")==0)
  1353. len_name = 0; // don't remove characters from numeric match strings
  1354. for(ix=0; ix<n_rules; ix++)
  1355. {
  1356. p = rules[ix];
  1357. len1 = strlen(p) + 1; // phoneme string
  1358. p3 = &p[len1];
  1359. p2 = p3 + len_name; // remove group name from start of match string
  1360. len2 = strlen(p2);
  1361. nextchar_count[(unsigned char)(p2[0])]++; // the next byte after the group name
  1362. if((common[0] != 0) && (strcmp(p,common)==0))
  1363. {
  1364. fwrite(p2,len2,1,f_out);
  1365. fputc(0,f_out); // no phoneme string, it's the same as previous rule
  1366. }
  1367. else
  1368. {
  1369. if((ix < n_rules-1) && (strcmp(p,rules[ix+1])==0))
  1370. {
  1371. common = rules[ix]; // phoneme string is same as next, set as common
  1372. fputc(RULE_PH_COMMON,f_out);
  1373. }
  1374. fwrite(p2,len2,1,f_out);
  1375. fputc(RULE_PHONEMES,f_out);
  1376. fwrite(p,len1,1,f_out);
  1377. }
  1378. }
  1379. #ifdef LIST_GROUP_INFO
  1380. for(ix=32; ix<256; ix++)
  1381. {
  1382. if(nextchar_count[ix] > 30)
  1383. printf("Group %s %c %d\n",name,ix,nextchar_count[ix]);
  1384. }
  1385. #endif
  1386. } // end of output_rule_group
  1387. static int compile_lettergroup(char *input, FILE *f_out)
  1388. {//=====================================================
  1389. char *p;
  1390. char *p_start;
  1391. int group;
  1392. int ix;
  1393. int n_items;
  1394. int length;
  1395. int max_length = 0;
  1396. #define N_LETTERGP_ITEMS 200
  1397. char *items[N_LETTERGP_ITEMS];
  1398. char item_length[N_LETTERGP_ITEMS];
  1399. p = input;
  1400. if(!isdigit(p[0]) || !isdigit(p[1]))
  1401. {
  1402. fprintf(f_log,"%5d: Expected 2 digits after '.L'\n",linenum);
  1403. error_count++;
  1404. return(1);
  1405. }
  1406. group = atoi(&p[0]);
  1407. if(group >= N_LETTER_GROUPS)
  1408. {
  1409. fprintf(f_log,"%5d: lettergroup out of range (01-%.2d)\n",linenum,N_LETTER_GROUPS-1);
  1410. error_count++;
  1411. return(1);
  1412. }
  1413. while(!isspace2(*p)) p++;
  1414. fputc(RULE_GROUP_START,f_out);
  1415. fputc(RULE_LETTERGP2,f_out);
  1416. fputc(group + 'A', f_out);
  1417. if(letterGroupsDefined[group] != 0)
  1418. {
  1419. fprintf(f_log,"%5d: lettergroup L%.2d is already defined\n",linenum,group);
  1420. error_count++;
  1421. }
  1422. letterGroupsDefined[group] = 1;
  1423. n_items = 0;
  1424. while(n_items < N_LETTERGP_ITEMS)
  1425. {
  1426. while(isspace2(*p)) p++;
  1427. if(*p == 0)
  1428. break;
  1429. items[n_items] = p_start = p;
  1430. while((*p & 0xff) > ' ')
  1431. {
  1432. if (*p == '_') *p = ' '; // allow '_' for word break
  1433. p++;
  1434. }
  1435. *p++ = 0;
  1436. length = p - p_start;
  1437. if(length > max_length)
  1438. max_length = length;
  1439. item_length[n_items++] = length;
  1440. }
  1441. // write out the items, longest first
  1442. while(max_length > 1)
  1443. {
  1444. for(ix=0; ix < n_items; ix++)
  1445. {
  1446. if(item_length[ix] == max_length)
  1447. {
  1448. fwrite(items[ix],1,max_length,f_out);
  1449. }
  1450. }
  1451. max_length--;
  1452. }
  1453. fputc(RULE_GROUP_END,f_out);
  1454. return(0);
  1455. }
  1456. static int compile_dictrules(FILE *f_in, FILE *f_out, char *fname_temp)
  1457. {//====================================================================
  1458. char *prule;
  1459. unsigned char *p;
  1460. int ix;
  1461. int c;
  1462. int gp;
  1463. FILE *f_temp;
  1464. int n_rules=0;
  1465. int count=0;
  1466. int different;
  1467. int wc;
  1468. const char *prev_rgroup_name;
  1469. unsigned int char_code;
  1470. int compile_mode=0;
  1471. char *buf;
  1472. char buf1[500];
  1473. char *rules[N_RULES];
  1474. int n_rgroups = 0;
  1475. int n_groups3 = 0;
  1476. RGROUP rgroup[N_RULE_GROUP2];
  1477. linenum = 0;
  1478. group_name[0] = 0;
  1479. if((f_temp = fopen_log(fname_temp,"wb")) == NULL)
  1480. return(1);
  1481. for(;;)
  1482. {
  1483. linenum++;
  1484. buf = fgets(buf1,sizeof(buf1),f_in);
  1485. if(buf != NULL)
  1486. {
  1487. if((p = (unsigned char *)strstr(buf,"//")) != NULL)
  1488. *p = 0;
  1489. if(buf[0] == '\r') buf++; // ignore extra \r in \r\n
  1490. }
  1491. if((buf == NULL) || (buf[0] == '.'))
  1492. {
  1493. // next .group or end of file, write out the previous group
  1494. if(n_rules > 0)
  1495. {
  1496. strcpy(rgroup[n_rgroups].name,group_name);
  1497. rgroup[n_rgroups].group3_ix = group3_ix;
  1498. rgroup[n_rgroups].start = ftell(f_temp);
  1499. output_rule_group(f_temp,n_rules,rules,group_name);
  1500. rgroup[n_rgroups].length = ftell(f_temp) - rgroup[n_rgroups].start;
  1501. n_rgroups++;
  1502. count += n_rules;
  1503. }
  1504. n_rules = 0;
  1505. if(compile_mode == 2)
  1506. {
  1507. // end of the character replacements section
  1508. fwrite(&n_rules,1,4,f_out); // write a zero word to terminate the replacemenmt list
  1509. compile_mode = 0;
  1510. }
  1511. if(buf == NULL) break; // end of file
  1512. if(memcmp(buf,".L",2)==0)
  1513. {
  1514. compile_lettergroup(&buf[2], f_out);
  1515. continue;
  1516. }
  1517. if(memcmp(buf,".replace",8)==0)
  1518. {
  1519. compile_mode = 2;
  1520. fputc(RULE_GROUP_START,f_out);
  1521. fputc(RULE_REPLACEMENTS,f_out);
  1522. // advance to next word boundary
  1523. while((ftell(f_out) & 3) != 0)
  1524. fputc(0,f_out);
  1525. }
  1526. if(memcmp(buf,".group",6)==0)
  1527. {
  1528. compile_mode = 1;
  1529. p = (unsigned char *)&buf[6];
  1530. while((p[0]==' ') || (p[0]=='\t')) p++; // Note: Windows isspace(0xe1) gives TRUE !
  1531. ix = 0;
  1532. while((*p > ' ') && (ix < LEN_GROUP_NAME))
  1533. group_name[ix++] = *p++;
  1534. group_name[ix]=0;
  1535. group3_ix = 0;
  1536. if(sscanf(group_name,"0x%x",&char_code)==1)
  1537. {
  1538. // group character is given as a character code (max 16 bits)
  1539. p = (unsigned char *)group_name;
  1540. if(char_code > 0x100)
  1541. {
  1542. *p++ = (char_code >> 8);
  1543. }
  1544. *p++ = char_code;
  1545. *p = 0;
  1546. }
  1547. else
  1548. {
  1549. if(translator->letter_bits_offset > 0)
  1550. {
  1551. utf8_in(&wc, group_name);
  1552. if(((ix = (wc - translator->letter_bits_offset)) >= 0) && (ix < 128))
  1553. {
  1554. group3_ix = ix+1; // not zero
  1555. }
  1556. }
  1557. }
  1558. if((group3_ix == 0) && (strlen(group_name) > 2))
  1559. {
  1560. if(utf8_in(&c,group_name) < 2)
  1561. {
  1562. fprintf(f_log,"%5d: Group name longer than 2 bytes (UTF8)",linenum);
  1563. error_count++;
  1564. }
  1565. group_name[2] = 0;
  1566. }
  1567. }
  1568. continue;
  1569. }
  1570. switch(compile_mode)
  1571. {
  1572. case 1: // .group
  1573. prule = compile_rule(buf);
  1574. if((prule != NULL) && (n_rules < N_RULES))
  1575. {
  1576. rules[n_rules++] = prule;
  1577. }
  1578. break;
  1579. case 2: // .replace
  1580. {
  1581. int replace1;
  1582. int replace2;
  1583. char *p;
  1584. p = buf;
  1585. replace1 = 0;
  1586. replace2 = 0;
  1587. while(isspace2(*p)) p++;
  1588. ix = 0;
  1589. while((unsigned char)(*p) > 0x20) // not space or zero-byte
  1590. {
  1591. p += utf8_in(&c,p);
  1592. replace1 += (c << ix);
  1593. ix += 16;
  1594. }
  1595. while(isspace2(*p)) p++;
  1596. ix = 0;
  1597. while((unsigned char)(*p) > 0x20)
  1598. {
  1599. p += utf8_in(&c,p);
  1600. replace2 += (c << ix);
  1601. ix += 16;
  1602. }
  1603. if(replace1 != 0)
  1604. {
  1605. Write4Bytes(f_out,replace1); // write as little-endian
  1606. Write4Bytes(f_out,replace2); // if big-endian, reverse the bytes in LoadDictionary()
  1607. }
  1608. }
  1609. break;
  1610. }
  1611. }
  1612. fclose(f_temp);
  1613. qsort((void *)rgroup,n_rgroups,sizeof(rgroup[0]),(int (__cdecl *)(const void *,const void *))rgroup_sorter);
  1614. if((f_temp = fopen(fname_temp,"rb"))==NULL)
  1615. return(2);
  1616. prev_rgroup_name = "\n";
  1617. for(gp = 0; gp < n_rgroups; gp++)
  1618. {
  1619. fseek(f_temp,rgroup[gp].start,SEEK_SET);
  1620. if((different = strcmp(rgroup[gp].name, prev_rgroup_name)) != 0)
  1621. {
  1622. // not the same as the previous group
  1623. if(gp > 0)
  1624. fputc(RULE_GROUP_END,f_out);
  1625. fputc(RULE_GROUP_START,f_out);
  1626. if(rgroup[gp].group3_ix != 0)
  1627. {
  1628. n_groups3++;
  1629. fputc(1,f_out);
  1630. fputc(rgroup[gp].group3_ix, f_out);
  1631. }
  1632. else
  1633. {
  1634. fprintf(f_out, "%s", prev_rgroup_name = rgroup[gp].name);
  1635. }
  1636. fputc(0,f_out);
  1637. }
  1638. for(ix=rgroup[gp].length; ix>0; ix--)
  1639. {
  1640. c = fgetc(f_temp);
  1641. fputc(c,f_out);
  1642. }
  1643. if(different)
  1644. {
  1645. }
  1646. }
  1647. fputc(RULE_GROUP_END,f_out);
  1648. fputc(0,f_out);
  1649. fclose(f_temp);
  1650. remove(fname_temp);
  1651. fprintf(f_log,"\t%d rules, %d groups (%d)\n\n",count,n_rgroups,n_groups3);
  1652. return(0);
  1653. } // end of compile_dictrules
  1654. int CompileDictionary(const char *dsource, const char *dict_name, FILE *log, char *fname_err, int flags)
  1655. {//=====================================================================================================
  1656. // fname: space to write the filename in case of error
  1657. // flags: bit 0: include source line number information, for debug purposes.
  1658. FILE *f_in;
  1659. FILE *f_out;
  1660. int offset_rules=0;
  1661. int value;
  1662. char fname_in[sizeof(path_home)+45];
  1663. char fname_out[sizeof(path_home)+15];
  1664. char fname_temp[sizeof(path_home)+15];
  1665. char path[sizeof(path_home)+40]; // path_dsource+20
  1666. error_count = 0;
  1667. error_need_dictionary = 0;
  1668. memset(letterGroupsDefined,0,sizeof(letterGroupsDefined));
  1669. debug_flag = flags & 1;
  1670. if(dsource == NULL)
  1671. dsource = "";
  1672. f_log = log;
  1673. //f_log = fopen("log2.txt","w");
  1674. if(f_log == NULL)
  1675. f_log = stderr;
  1676. // try with and without '.txt' extension
  1677. sprintf(path,"%s%s_",dsource,dict_name);
  1678. sprintf(fname_in,"%srules.txt",path);
  1679. if((f_in = fopen(fname_in,"r")) == NULL)
  1680. {
  1681. sprintf(fname_in,"%srules",path);
  1682. if((f_in = fopen_log(fname_in,"r")) == NULL)
  1683. {
  1684. if(fname_err)
  1685. strcpy(fname_err,fname_in);
  1686. return(-1);
  1687. }
  1688. }
  1689. sprintf(fname_out,"%s%c%s_dict",path_home,PATHSEP,dict_name);
  1690. if((f_out = fopen_log(fname_out,"wb+")) == NULL)
  1691. {
  1692. if(fname_err)
  1693. strcpy(fname_err,fname_in);
  1694. return(-1);
  1695. }
  1696. sprintf(fname_temp,"%s%ctemp",path_home,PATHSEP);
  1697. value = N_HASH_DICT;
  1698. Write4Bytes(f_out,value);
  1699. Write4Bytes(f_out,offset_rules);
  1700. compile_dictlist_start();
  1701. fprintf(f_log,"Using phonemetable: '%s'\n",phoneme_tab_list[phoneme_tab_number].name);
  1702. compile_dictlist_file(path,"roots");
  1703. if(translator->langopts.listx)
  1704. {
  1705. compile_dictlist_file(path,"list");
  1706. compile_dictlist_file(path,"listx");
  1707. }
  1708. else
  1709. {
  1710. compile_dictlist_file(path,"listx");
  1711. compile_dictlist_file(path,"list");
  1712. }
  1713. compile_dictlist_file(path,"extra");
  1714. compile_dictlist_end(f_out);
  1715. offset_rules = ftell(f_out);
  1716. fprintf(f_log,"Compiling: '%s'\n",fname_in);
  1717. compile_dictrules(f_in,f_out,fname_temp);
  1718. fclose(f_in);
  1719. fseek(f_out,4,SEEK_SET);
  1720. Write4Bytes(f_out,offset_rules);
  1721. fclose(f_out);
  1722. LoadDictionary(translator, dict_name, 0);
  1723. return(error_count);
  1724. } // end of compile_dictionary