1 // Package enzyme implements extracellular digestion — decomposing raw input
2 // into typed elements that can bond into the lattice.
3 //
4 // Each enzyme is substrate-specific. Cellulase for cellulose, protease for
5 // protein. The type of each emitted element is determined by the enzyme's
6 // own type system.
7 package enzyme
8 9 import (
10 "bufio"
11 "io"
12 "strings"
13 "unicode"
14 15 "git.mleku.dev/mleku/dendrite/pkg/axiom"
16 )
17 18 // Enzyme decomposes raw substrate into typed elements.
19 type Enzyme interface {
20 // CanDigest reports whether this enzyme can process the given sample.
21 CanDigest(sample []byte) bool
22 23 // Digest reads from the substrate and emits typed elements.
24 // The channel is closed when the substrate is exhausted.
25 Digest(r io.Reader) <-chan axiom.Element
26 }
27 28 // element is the concrete Element type emitted by enzymes.
29 type element struct {
30 tag string
31 val any
32 }
33 34 func (e element) Type() string { return e.tag }
35 func (e element) Value() any { return e.val }
36 37 // Elem creates an element with the given type tag and value.
38 // If the value is a string, the element carries hexagram-encoded tokens.
39 func Elem(tag string, val any) axiom.Element {
40 if s, ok := val.(string); ok {
41 return newHexElement(tag, s)
42 }
43 return element{tag, val}
44 }
45 46 // hexElement carries hexagram-encoded value alongside the raw value.
47 // It satisfies both axiom.Element and axiom.HexagramElement.
48 type hexElement struct {
49 tag string
50 val any
51 tokens []uint8
52 origLen int
53 }
54 55 func (e hexElement) Type() string { return e.tag }
56 func (e hexElement) Value() any { return e.val }
57 func (e hexElement) HexTokens() []uint8 { return e.tokens }
58 func (e hexElement) OrigLen() int { return e.origLen }
59 60 // HexElem creates an element with hexagram-encoded value.
61 // The raw string is preserved as Value() for backward compatibility;
62 // the hexagram tokens are available via the HexagramElement interface.
63 func HexElem(tag string, val string) axiom.Element {
64 return newHexElement(tag, val)
65 }
66 67 // newHexElement is the package-internal constructor for hexagram-encoded elements.
68 func newHexElement(tag string, val string) hexElement {
69 data := []byte(val)
70 tokens := encodeStringToHex(data)
71 return hexElement{tag, val, tokens, len(data)}
72 }
73 74 // encodeStringToHex converts raw bytes to 6-bit hexagram tokens.
75 // Every 3 bytes produce 4 tokens (24 bits = 4 × 6 bits).
76 func encodeStringToHex(data []byte) []uint8 {
77 if len(data) == 0 {
78 return nil
79 }
80 groups := (len(data) + 2) / 3
81 out := make([]uint8, groups*4)
82 for i := 0; i < len(data); i += 3 {
83 var block [3]byte
84 copy(block[:], data[i:min(i+3, len(data))])
85 bits := uint32(block[0])<<16 | uint32(block[1])<<8 | uint32(block[2])
86 j := (i / 3) * 4
87 out[j+0] = uint8((bits >> 18) & 0x3F)
88 out[j+1] = uint8((bits >> 12) & 0x3F)
89 out[j+2] = uint8((bits >> 6) & 0x3F)
90 out[j+3] = uint8(bits & 0x3F)
91 }
92 return out
93 }
94 95 // Text is an enzyme that decomposes UTF-8 text into typed tokens.
96 // Words, punctuation, and whitespace are emitted as separate elements.
97 type Text struct{}
98 99 // CanDigest returns true for any input — text is the universal substrate.
100 func (Text) CanDigest([]byte) bool { return true }
101 102 // Digest breaks the input into word, punct, and space elements.
103 func (Text) Digest(r io.Reader) <-chan axiom.Element {
104 ch := make(chan axiom.Element, 64)
105 go func() {
106 defer close(ch)
107 scanner := bufio.NewScanner(r)
108 scanner.Split(scanTokens)
109 for scanner.Scan() {
110 tok := scanner.Text()
111 tag := classifyToken(tok)
112 ch <- hexElement{tag, tok, encodeStringToHex([]byte(tok)), len(tok)}
113 }
114 }()
115 return ch
116 }
117 118 // classifyToken determines the type tag for a text token.
119 // Words are sub-classified by length bucket to create richer constraint
120 // envelopes. AI text tends toward uniform medium-length words; human text
121 // has more variation across length classes.
122 func classifyToken(tok string) string {
123 if len(tok) == 0 {
124 return "empty"
125 }
126 // Check first rune.
127 r := []rune(tok)[0]
128 switch {
129 case unicode.IsLetter(r) || unicode.IsDigit(r):
130 return wordLengthTag(tok)
131 case unicode.IsSpace(r):
132 return "space"
133 default:
134 return "punct"
135 }
136 }
137 138 // wordLengthTag classifies a word token by its length bucket.
139 // The buckets are chosen to capture stylometric variation:
140 //
141 // w1: 1 char (articles, pronouns: a, I)
142 // w2: 2-3 chars (common words: the, is, an, to, of)
143 // w3: 4-5 chars (core vocab: from, with, that, about)
144 // w4: 6-8 chars (content words: between, another, writing)
145 // w5: 9+ chars (formal/technical: restructured, acknowledging)
146 func wordLengthTag(tok string) string {
147 n := len([]rune(tok))
148 switch {
149 case n <= 1:
150 return "w1"
151 case n <= 3:
152 return "w2"
153 case n <= 5:
154 return "w3"
155 case n <= 8:
156 return "w4"
157 default:
158 return "w5"
159 }
160 }
161 162 // scanTokens is a bufio.SplitFunc that splits into words, whitespace runs,
163 // and individual punctuation characters.
164 func scanTokens(data []byte, atEOF bool) (advance int, token []byte, err error) {
165 if len(data) == 0 {
166 return 0, nil, nil
167 }
168 169 r := rune(data[0])
170 171 // Whitespace run.
172 if unicode.IsSpace(r) {
173 i := 0
174 for i < len(data) && unicode.IsSpace(rune(data[i])) {
175 i++
176 }
177 return i, data[:i], nil
178 }
179 180 // Word (letters and digits).
181 if unicode.IsLetter(r) || unicode.IsDigit(r) {
182 i := 0
183 for i < len(data) {
184 r := rune(data[i])
185 if !unicode.IsLetter(r) && !unicode.IsDigit(r) {
186 break
187 }
188 i++
189 }
190 if i == 0 && !atEOF {
191 return 0, nil, nil // need more data
192 }
193 return i, data[:i], nil
194 }
195 196 // Single punctuation character.
197 return 1, data[:1], nil
198 }
199 200 // RefElement wraps an element to mark it as reference material.
201 // Reference elements influence lattice topology during growth but
202 // are excluded from emitted output. Use IsRef to check.
203 type RefElement struct {
204 Inner axiom.Element
205 }
206 207 func (r RefElement) Type() string { return r.Inner.Type() }
208 func (r RefElement) Value() any { return r.Inner.Value() }
209 210 // HexTokens forwards to the inner element if it supports hexagram encoding.
211 func (r RefElement) HexTokens() []uint8 {
212 if h, ok := r.Inner.(interface{ HexTokens() []uint8 }); ok {
213 return h.HexTokens()
214 }
215 return nil
216 }
217 218 // IsRef reports whether an element is reference material.
219 func IsRef(e axiom.Element) bool {
220 _, ok := e.(RefElement)
221 return ok
222 }
223 224 // Lines is an enzyme that emits each line as a single element.
225 type Lines struct {
226 Tag string // type tag for emitted elements; defaults to "line"
227 }
228 229 func (e Lines) CanDigest([]byte) bool { return true }
230 231 func (e Lines) Digest(r io.Reader) <-chan axiom.Element {
232 tag := e.Tag
233 if tag == "" {
234 tag = "line"
235 }
236 ch := make(chan axiom.Element, 64)
237 go func() {
238 defer close(ch)
239 scanner := bufio.NewScanner(r)
240 for scanner.Scan() {
241 line := strings.TrimRight(scanner.Text(), "\r\n")
242 ch <- hexElement{tag, line, encodeStringToHex([]byte(line)), len(line)}
243 }
244 }()
245 return ch
246 }
247