english.go raw
1 package shadow
2
3 import (
4 "bufio"
5 "embed"
6 "fmt"
7 "strings"
8 "sync"
9 "unicode"
10 )
11
12 //go:embed cmudict.txt
13 var dictFS embed.FS
14
15 // ARPAbet → Shadow mapping.
16 //
17 // CMU uses stress digits on vowels: 0=no stress, 1=primary, 2=secondary.
18 // The digits are stripped before lookup here; stress is handled separately.
19 //
20 // Vowel mapping rationale (American English → 7-vowel Shadow system):
21 //
22 // AA /ɑ/ → A (father, hot)
23 // AE /æ/ → A (cat — no /æ/ in Shadow, collapses to open A)
24 // AH /ʌ/ → A (but, cup — schwa/strut vowel → A)
25 // AO /ɔ/ → O (thought, law)
26 // AW /aʊ/ → AU (diphthong: cow)
27 // AY /aɪ/ → AY (diphthong: eye — Y as glide)
28 // EH /ɛ/ → E (bed)
29 // ER /ɝ/ → R (syllabic r: bird, butter)
30 // EY /eɪ/ → EY (diphthong: day)
31 // IH /ɪ/ → I (bit — near-close, collapses to I)
32 // IY /i/ → I (see)
33 // OW /oʊ/ → O (go — diphthong simplified)
34 // OY /ɔɪ/ → OY (diphthong: boy)
35 // UH /ʊ/ → U (book — near-close, collapses to U)
36 // UW /u/ → U (moon)
37 var arpabetToShadow = map[string]string{
38 // Vowels (monophthongs)
39 "AA": A,
40 "AE": A,
41 "AH": A,
42 "AO": O,
43 "EH": E,
44 "ER": R,
45 "IH": I,
46 "IY": I,
47 "UH": U,
48 "UW": U,
49
50 // Diphthongs (decomposed into Shadow vowel sequences)
51 "AW": A + U,
52 "AY": A + Y,
53 "EY": E + Y,
54 "OW": O,
55 "OY": O + Y,
56
57 // Consonants
58 "B": B,
59 "CH": Ch,
60 "D": D,
61 "DH": Dh,
62 "F": F,
63 "G": G,
64 "HH": H,
65 "JH": Dzh,
66 "K": K,
67 "L": L,
68 "M": M,
69 "N": N,
70 "NG": Ng,
71 "P": P,
72 "R": R,
73 "S": S,
74 "SH": Sh,
75 "T": T,
76 "TH": Th,
77 "V": V,
78 "W": W,
79 "Y": Y,
80 "Z": Z,
81 "ZH": Zh,
82 }
83
84 // EnglishDict holds the parsed CMU dictionary.
85 type EnglishDict struct {
86 // words maps lowercase English words to their Shadow transliterations.
87 // Multiple pronunciations are stored with the first (most common) only.
88 words map[string]string
89 }
90
91 var (
92 defaultDict *EnglishDict
93 defaultDictOnce sync.Once
94 defaultDictErr error
95 )
96
97 // LoadEnglishDict loads the embedded CMU dictionary and builds the
98 // word → Shadow mapping.
99 func LoadEnglishDict() (*EnglishDict, error) {
100 defaultDictOnce.Do(func() {
101 defaultDict, defaultDictErr = loadDict()
102 })
103 return defaultDict, defaultDictErr
104 }
105
106 func loadDict() (*EnglishDict, error) {
107 f, err := dictFS.Open("cmudict.txt")
108 if err != nil {
109 return nil, fmt.Errorf("shadow: open embedded dict: %w", err)
110 }
111 defer f.Close()
112
113 d := &EnglishDict{
114 words: make(map[string]string, 140000),
115 }
116
117 sc := bufio.NewScanner(f)
118 for sc.Scan() {
119 line := sc.Text()
120 if len(line) == 0 || line[0] == ';' {
121 continue
122 }
123
124 // Format: "WORD PH1 PH2 PH3" (two-space separator)
125 // Alternate pronunciations: "WORD(2) PH1 PH2"
126 parts := strings.SplitN(line, " ", 2)
127 if len(parts) != 2 {
128 continue
129 }
130
131 word := strings.TrimSpace(parts[0])
132 phonemes := strings.TrimSpace(parts[1])
133
134 // Skip alternate pronunciations — keep only the first
135 if strings.Contains(word, "(") {
136 continue
137 }
138
139 // Normalize word to lowercase
140 word = strings.ToLower(word)
141
142 // Convert phoneme sequence to Shadow
143 shadow := phonemesToShadow(phonemes)
144 d.words[word] = shadow
145 }
146
147 if err := sc.Err(); err != nil {
148 return nil, fmt.Errorf("shadow: scan dict: %w", err)
149 }
150
151 return d, nil
152 }
153
154 // phonemesToShadow converts a CMU ARPAbet phoneme string to Shadow letters.
155 // Stress digits on vowels are used to place the accent mark.
156 func phonemesToShadow(phonemes string) string {
157 parts := strings.Fields(phonemes)
158 var out strings.Builder
159
160 for _, ph := range parts {
161 // Strip stress digit (0, 1, 2) from end of vowel phonemes
162 base := ph
163 stress := byte(0)
164 if len(ph) > 1 {
165 last := ph[len(ph)-1]
166 if last >= '0' && last <= '2' {
167 stress = last
168 base = ph[:len(ph)-1]
169 }
170 }
171
172 s, ok := arpabetToShadow[base]
173 if !ok {
174 // Unknown phoneme — skip
175 continue
176 }
177
178 out.WriteString(s)
179
180 // Place accent mark after primary-stressed vowels
181 if stress == '1' {
182 out.WriteString(Accent)
183 }
184 }
185
186 return out.String()
187 }
188
189 // FromEnglish transliterates English text into the Shadow Alphabet.
190 // Words are looked up in the CMU dictionary. Words not found are
191 // passed through unchanged (preserving case). Punctuation and
192 // whitespace pass through.
193 func (d *EnglishDict) FromEnglish(text string) string {
194 var out strings.Builder
195 out.Grow(len(text) * 2)
196
197 i := 0
198 for i < len(text) {
199 // Skip and copy whitespace
200 if text[i] == ' ' || text[i] == '\t' || text[i] == '\n' || text[i] == '\r' {
201 out.WriteByte(text[i])
202 i++
203 continue
204 }
205
206 // Collect a word (letters and apostrophes)
207 start := i
208 for i < len(text) && (isWordChar(rune(text[i]))) {
209 i++
210 }
211
212 if i > start {
213 word := text[start:i]
214 lower := strings.ToLower(word)
215
216 if shadow, ok := d.words[lower]; ok {
217 out.WriteString(shadow)
218 } else {
219 // Not in dictionary — pass through unchanged
220 out.WriteString(word)
221 }
222 continue
223 }
224
225 // Non-word character (punctuation, digits) — pass through
226 out.WriteByte(text[i])
227 i++
228 }
229
230 return out.String()
231 }
232
233 func isWordChar(r rune) bool {
234 return unicode.IsLetter(r) || r == '\''
235 }
236
237 // Lookup returns the Shadow transliteration for a single English word,
238 // or empty string and false if not found.
239 func (d *EnglishDict) Lookup(word string) (string, bool) {
240 s, ok := d.words[strings.ToLower(word)]
241 return s, ok
242 }
243
244 // Size returns the number of words in the dictionary.
245 func (d *EnglishDict) Size() int {
246 return len(d.words)
247 }
248