diff --git a/models/tts/kokoro/coreml/g2p/japanese/README.md b/models/tts/kokoro/coreml/g2p/japanese/README.md new file mode 100644 index 0000000..2d33fe9 --- /dev/null +++ b/models/tts/kokoro/coreml/g2p/japanese/README.md @@ -0,0 +1,45 @@ +# Kokoro ANE Japanese frontend assets (FluidAudio #914) + +FluidAudio's Japanese text frontend (`Sources/FluidAudio/TTS/KokoroAne/G2P/Japanese/`) +is a Swift port of Misaki's Cutlet: a MeCab-compatible Viterbi tokenizer over +`unidic-lite` plus Cutlet's kana → IPA rules. The tokenizer reads the standard +MeCab binary dictionary layout, so the assets are unidic-lite's own files, +trimmed to the three feature fields the frontend uses. + +## Build the assets + +```bash +uv run --with unidic-lite python convert_unidic_lite.py \ + "$(uv run --with unidic-lite python -c 'import unidic_lite, os; print(os.path.join(os.path.dirname(unidic_lite.__file__), "dicdir"))')" \ + out/ +``` + +`convert_unidic_lite.py` keeps the double array and token table byte-for-byte, +rewrites every feature string to `pos1,pron,kana` (UniDic indices 0, 9, 17) +and applies the same trim to `unk.dic`; `matrix.bin` and `char.bin` are copied. +Sizes: `sys.dic` 188 MB → 41 MB, `matrix.bin` 71.5 MB, `char.bin` 0.3 MB, +`unk.dic` 4 KB. Add Misaki's `ja_words.txt` (`misaki/data/ja_words.txt`, 2 MB). + +## Validate + +`mecab_reference.py` is a pure-Python reader of the same binary layout with a +MeCab-style Viterbi; run it against fugashi on any sentences to confirm the +trimmed dictionary tokenizes identically: + +```bash +uv run --with unidic-lite --with fugashi python mecab_reference.py out/ '今日は良い天気です。' '私は日本語を勉強しています。' +``` + +It prints `SAME`/`DIFF` per sentence (surface, pron, kana, unknown flag and +character category must all match). The Swift `JapaneseTokenizer` mirrors this +script. + +## Publish + +Upload `sys.dic`, `unk.dic`, `char.bin`, `matrix.bin`, `ja_words.txt` to +`FluidInference/kokoro-82m-coreml` under `ANE-ja/assets/` (the Mandarin tables +live under `ANE-zh/assets/`). FluidAudio downloads them into `/g2p/` +on the first plain-text Japanese synthesis. + +Licenses: unidic-lite / UniDic (BSD), misaki (Apache-2.0), cutlet (MIT), +num2kana (MIT) — see `ThirdPartyLicenses/JapaneseG2P-LICENSE.md` in FluidAudio. diff --git a/models/tts/kokoro/coreml/g2p/japanese/convert_unidic_lite.py b/models/tts/kokoro/coreml/g2p/japanese/convert_unidic_lite.py new file mode 100644 index 0000000..9276442 --- /dev/null +++ b/models/tts/kokoro/coreml/g2p/japanese/convert_unidic_lite.py @@ -0,0 +1,37 @@ +"""Trim a MeCab dictionary (unidic-lite) to the fields the Kokoro Japanese frontend needs. + +Keeps the double array and token table byte-for-byte; rewrites the feature blob so each +entry's feature string is `pos1,pron,kana` (UniDic indices 0, 9, 17). Applies the same +trim to unk.dic. matrix.bin and char.bin are copied unchanged. Output layout is the +standard MeCab sys.dic layout, so the same reader handles both. +""" +import struct, os, sys, shutil +def trim(src, dst, fields=(0, 9, 17)): + b = open(src, "rb").read() + head = list(struct.unpack("<10I", b[:40])); charset = b[40:72] + lexsize, lsize, rsize, dsize, tsize, fsize = head[3:9] + off = 72 + darts = b[off:off+dsize]; off += dsize + tokens = bytearray(b[off:off+tsize]); off += tsize + feats = b[off:off+fsize] + new_feats = bytearray(); cache = {} + for i in range(tsize // 16): + lc, rc, posid, wcost, foff, comp = struct.unpack_from(" {b_/1e6:.1f} MB ({n} unique features)") + for name in ("matrix.bin", "char.bin"): + shutil.copy(f"{src}/{name}", f"{dst}/{name}"); print(f"{name}: {os.path.getsize(f'{dst}/{name}')/1e6:.1f} MB (copied)") diff --git a/models/tts/kokoro/coreml/g2p/japanese/mecab_reference.py b/models/tts/kokoro/coreml/g2p/japanese/mecab_reference.py new file mode 100644 index 0000000..b4caab0 --- /dev/null +++ b/models/tts/kokoro/coreml/g2p/japanese/mecab_reference.py @@ -0,0 +1,138 @@ +"""Reference MeCab-format reader + Viterbi, to validate the dictionary format before the Swift port. +Reads unidic-lite's sys.dic / matrix.bin / char.bin / unk.dic directly.""" +import struct, os, sys, unicodedata +class Dic: + def __init__(self, path): + b = open(path, "rb").read() + (self.magic, self.version, self.type, self.lexsize, self.lsize, self.rsize, + self.dsize, self.tsize, self.fsize, _) = struct.unpack("<10I", b[:40]) + off = 72 + self.darts = b[off:off+self.dsize]; off += self.dsize + self.tokens = b[off:off+self.tsize]; off += self.tsize + self.features = b[off:off+self.fsize] + def unit(self, i): + base, check = struct.unpack_from("> 8, value & 0xff + for k in range(count): + lc, rc, posid, wcost, foff = self.token(idx + k) + yield length, lc, rc, wcost, foff +class CharInfo: + def __init__(self, path): + b = open(path, "rb").read() + n = struct.unpack_from(" 0xFFFF: cp = 0 # MeCab maps > BMP to DEFAULT? (it uses the last code point info); keep simple + v = struct.unpack_from("> 18) & 0xFF, length=(v >> 26) & 0xF, group=(v >> 30) & 1, invoke=(v >> 31) & 1) +class Matrix: + def __init__(self, path): + b = open(path, "rb").read(); self.l, self.r = struct.unpack_from(" list of (cost, rid, lid?, feature, backptr) + # node: (end, cost_so_far, rid, start, feature_off, is_unk, char_type, prev_node) + ends = {0: [(0, 0, None, None, None, None, None, None)]} + for k, (off, l, ch) in enumerate(cps): + if off not in ends: continue + prev_nodes = ends[off] + key = raw[off:] + cands = [] + for length, lc, rc, wcost, foff in dic.lookup(key): + cands.append((length, lc, rc, wcost, dic.feature(foff), False)) + ci = chars.info(ord(ch)) + cat = ci["default_type"]; group = ci["group"]; invoke = ci["invoke"]; maxlen = ci["length"] + # unknown word candidates (MeCab: invoke if no dictionary hit or invoke flag) + if not cands or invoke: + cat_id = ci["default_type"] + name = chars.names[cat_id] + uentries = list(unk.lookup(name.encode())) + lens = set() + if group: + # extend while the same category bit is set + j = k; total = 0 + while j < len(cps) and (chars.info(ord(cps[j][2]))["type"] >> cat_id) & 1: + total += cps[j][1]; j += 1 + if maxlen and (j - k) > maxlen: break + lens.add(total) + for m in range(1, (maxlen or 0) + 1): + if k + m <= len(cps) and all((chars.info(ord(cps[k+t][2]))["type"] >> cat_id) & 1 for t in range(m)): + lens.add(sum(cps[k+t][1] for t in range(m))) + if not lens: lens.add(l) + for length in lens: + for _, lc, rc, wcost, foff in uentries: + cands.append((length, lc, rc, wcost, unk.feature(foff), True)) + for length, lc, rc, wcost, feat, is_unk in cands: + end = off + length + bestc, bestp = INF, None + for pn in prev_nodes: + pcost, prid = pn[1], pn[2] + c = pcost + wcost + (matrix.cost(prid, lc) if prid is not None else 0) + if c < bestc: bestc, bestp = c, pn + node = (end, bestc, rc, off, feat, is_unk, cat, bestp) + ends.setdefault(end, []).append(node) + # EOS + final = min(ends.get(n, []), key=lambda nd: nd[1] + matrix.cost(nd[2], 0), default=None) + out = [] + nd = final + while nd is not None and nd[3] is not None: + out.append(nd); nd = nd[7] + out.reverse() + return [(raw[nd[3]:nd[0]].decode(), nd[4], nd[5], nd[6]) for nd in out] +if __name__ == "__main__": + base = sys.argv[1] + dic = Dic(f"{base}/sys.dic"); unk = Dic(f"{base}/unk.dic"); chars = CharInfo(f"{base}/char.bin"); matrix = Matrix(f"{base}/matrix.bin") + from fugashi import Tagger + tagger = Tagger() + mism = 0 + for text in sys.argv[2:]: + toks = tokenize(text, dic, unk, chars, matrix) + ref = [(w.surface, w.feature.pron or "", w.feature.kana or "", w.is_unk, w.char_type) for w in tagger(text)] + mine = [] + for s, f, u, ct in toks: + fs = f.split(","); pron, kana = (fs[9], fs[17]) if len(fs) > 17 else (fs[1], fs[2]) if len(fs) == 3 else ("", "") + mine.append((s, pron, kana, u, ct)) + ok = mine == ref + print(("SAME " if ok else "DIFF ") + text) + if not ok: + mism += 1; print(" mine:", mine); print(" ref :", ref) + print("mismatching sentences:", mism, "of", len(sys.argv) - 2)