mindsicle.go raw
1 // Package mindsicle implements the frozen lattice — the full graph state
2 // serialized for persistence and bootstrapping.
3 //
4 // A mindsicle is a lossless snapshot of the lattice: every node, edge,
5 // occupant, candidate, constraint tag, hex state, permutation, projection,
6 // and age. The spore compresses this to statistics; the mindsicle preserves
7 // the actual crystal.
8 //
9 // Freeze captures a live lattice into a Mindsicle. Thaw reconstitutes
10 // the lattice from the frozen state. The bootstrapper reads a mindsicle,
11 // thaws it, emits Go source, compiles, and runs the result.
12 package mindsicle
13
14 import (
15 "encoding/json"
16 "fmt"
17 "io"
18 "log"
19 "time"
20
21 "git.mleku.dev/mleku/dendrite/pkg/axiom"
22 "git.mleku.dev/mleku/dendrite/pkg/lattice"
23 "git.mleku.dev/mleku/dendrite/pkg/ratio"
24 "git.mleku.dev/mleku/dendrite/pkg/spore"
25 "git.mleku.dev/mleku/dendrite/pkg/state"
26 )
27
28 // Version is the current mindsicle format version.
29 const Version = 1
30
31 // Mindsicle is a frozen lattice — the complete graph state serialized.
32 type Mindsicle struct {
33 Spore *spore.Spore `json:"spore"`
34 Nodes []NodeRecord `json:"nodes"`
35 Version int `json:"version"`
36 FrozenAt time.Time `json:"frozen_at"`
37 }
38
39 // NodeRecord is the serialized form of a single lattice node.
40 type NodeRecord struct {
41 ID uint64 `json:"id"`
42 Tags []string `json:"constraints"`
43 Occupant *ElementRecord `json:"occupant,omitempty"`
44 Candidates []ElementRecord `json:"candidates,omitempty"`
45 Neighbors []uint64 `json:"neighbors"`
46 Hex uint8 `json:"hex"`
47 LockIn ratio.Ratio `json:"lock_in"`
48 BondCount int `json:"bond_count"`
49 Perm uint8 `json:"perm"`
50 ProjVertex uint8 `json:"proj_vertex"`
51 ProjKey uint8 `json:"proj_key"`
52 ProjPath uint16 `json:"proj_path"`
53 Age uint8 `json:"age"`
54 }
55
56 // ElementRecord is the serialized form of an element.
57 type ElementRecord struct {
58 Type string `json:"type"`
59 Value string `json:"value"`
60 }
61
62 // frozenElement implements axiom.Element for deserialized elements.
63 type frozenElement struct {
64 typ string
65 val string
66 }
67
68 func (e frozenElement) Type() string { return e.typ }
69 func (e frozenElement) Value() any { return e.val }
70
71 // Freeze captures a live lattice into a Mindsicle.
72 // If sp is nil, no spore is stored (Thaw doesn't need it).
73 func Freeze(l *lattice.Lattice, sp *spore.Spore) *Mindsicle {
74 nodes := l.Nodes()
75 records := make([]NodeRecord, len(nodes))
76
77 for i, n := range nodes {
78 rec := NodeRecord{
79 ID: uint64(n.ID()),
80 Hex: uint8(n.Hexagram()),
81 LockIn: n.LockIn(),
82 BondCount: n.BondCount(),
83 Perm: n.Permutation(),
84 ProjVertex: n.ProjectionVertex(),
85 ProjKey: n.ProjectionKey(),
86 ProjPath: n.ProjectionPath(),
87 Age: n.Age(),
88 }
89
90 // Constraint tags.
91 for _, c := range n.Constraints() {
92 rec.Tags = append(rec.Tags, c.Tag())
93 }
94
95 // Occupant.
96 if n.Occupied() {
97 e := n.Occupant()
98 val := ""
99 if e.Value() != nil {
100 val = fmt.Sprintf("%v", e.Value())
101 }
102 rec.Occupant = &ElementRecord{
103 Type: e.Type(),
104 Value: val,
105 }
106 }
107
108 // Candidates.
109 for _, c := range n.Candidates() {
110 val := ""
111 if c.Value() != nil {
112 val = fmt.Sprintf("%v", c.Value())
113 }
114 rec.Candidates = append(rec.Candidates, ElementRecord{
115 Type: c.Type(),
116 Value: val,
117 })
118 }
119
120 // Neighbors (as IDs).
121 for _, nb := range n.Neighbors() {
122 rec.Neighbors = append(rec.Neighbors, uint64(nb.ID()))
123 }
124
125 records[i] = rec
126 }
127
128 return &Mindsicle{
129 Spore: sp,
130 Nodes: records,
131 Version: Version,
132 FrozenAt: time.Now(),
133 }
134 }
135
136 // Thaw reconstitutes a live lattice from a frozen Mindsicle.
137 //
138 // constraintFactory converts tag strings back into Constraint instances.
139 // The thaw proceeds in four phases:
140 // 1. Create nodes with constraints
141 // 2. Connect neighbors (dedup bidirectional pairs)
142 // 3. ForceOccupant + candidates
143 // 4. Restore age, permutation, projection, hex energy/outer
144 func (m *Mindsicle) Thaw(constraintFactory func(string) axiom.Constraint) *lattice.Lattice {
145 l := lattice.New()
146
147 if len(m.Nodes) == 0 {
148 return l
149 }
150
151 // Phase 1: Create all nodes with their constraints.
152 nodeByID := make(map[uint64]*lattice.Node, len(m.Nodes))
153 for _, rec := range m.Nodes {
154 constraints := make([]axiom.Constraint, len(rec.Tags))
155 for j, tag := range rec.Tags {
156 constraints[j] = constraintFactory(tag)
157 }
158 n := l.AddNode(constraints)
159 nodeByID[uint64(n.ID())] = n
160 }
161
162 // Phase 2: Connect neighbors. Only connect when nbID > rec.ID to
163 // deduplicate bidirectional edges without any map allocation.
164 for _, rec := range m.Nodes {
165 for _, nbID := range rec.Neighbors {
166 if nbID > rec.ID {
167 na, nb := nodeByID[rec.ID], nodeByID[nbID]
168 if na != nil && nb != nil {
169 l.Connect(na, nb)
170 }
171 }
172 }
173 }
174
175 // Phase 3: Place occupants and candidates. Uses ForceOccupant to
176 // bypass constraint checking — the frozen state is pre-validated.
177 for _, rec := range m.Nodes {
178 n := nodeByID[rec.ID]
179 if n == nil {
180 continue
181 }
182
183 if rec.Occupant != nil {
184 elem := frozenElement{typ: rec.Occupant.Type, val: rec.Occupant.Value}
185 n.ForceOccupant(elem, rec.BondCount)
186 }
187
188 for _, c := range rec.Candidates {
189 elem := frozenElement{typ: c.Type, val: c.Value}
190 n.AddCandidate(elem)
191 }
192 }
193
194 // Phase 4: Restore age, permutation, projection, hex state, lock-in.
195 for _, rec := range m.Nodes {
196 n := nodeByID[rec.ID]
197 if n == nil {
198 continue
199 }
200
201 n.RestoreAge(rec.Age)
202 n.SetPermutation(rec.Perm)
203 n.SetProjection(rec.ProjVertex, rec.ProjKey, rec.ProjPath)
204
205 // Restore energy bit and outer trigram from the frozen hexagram.
206 hex := state.Hexagram(rec.Hex)
207 n.SetEnergy(hex.Inner().Energy())
208 n.SetOuterTrigram(hex.Outer())
209
210 // Restore the exact trained lock-in ratio. ForceOccupant sets a
211 // simple bondCount/1, but the frozen ratio preserves the real value.
212 n.RestoreLockIn(rec.LockIn)
213 }
214
215 return l
216 }
217
218 // WriteTo serializes the mindsicle to JSON.
219 func (m *Mindsicle) WriteTo(w io.Writer) (int64, error) {
220 data, err := json.MarshalIndent(m, "", " ")
221 if err != nil {
222 return 0, err
223 }
224 n, err := w.Write(data)
225 return int64(n), err
226 }
227
228 // ReadMindsicle deserializes a mindsicle from JSON.
229 func ReadMindsicle(r io.Reader) (*Mindsicle, error) {
230 data, err := io.ReadAll(r)
231 if err != nil {
232 return nil, err
233 }
234 var m Mindsicle
235 if err := json.Unmarshal(data, &m); err != nil {
236 return nil, err
237 }
238 return &m, nil
239 }
240
241 // StreamThawJSON reads a mindsicle JSON file using a streaming decoder,
242 // processing one NodeRecord at a time. This avoids the double allocation
243 // of io.ReadAll + json.Unmarshal that makes the non-streaming path OOM
244 // on large lattices (~4GB JSON → ~20GB peak with full materialization).
245 //
246 // Memory profile: ~1.6GB buffer (neighbor IDs + node state for deferred
247 // phases) + lattice itself. The 3.9GB JSON is never fully in memory.
248 func StreamThawJSON(r io.Reader, cf func(string) axiom.Constraint) (*lattice.Lattice, error) {
249 dec := json.NewDecoder(r)
250
251 // Read opening '{'.
252 if _, err := dec.Token(); err != nil {
253 return nil, fmt.Errorf("expected opening brace: %w", err)
254 }
255
256 // Compact per-node state buffer for deferred phases.
257 type nodeState struct {
258 neighborIDs []uint64
259 lockIn ratio.Ratio
260 hex uint8
261 perm uint8
262 projVertex uint8
263 projKey uint8
264 projPath uint16
265 age uint8
266 bondCount int
267 occType string
268 occValue string
269 candidates []ElementRecord
270 }
271
272 l := lattice.New()
273 var states []nodeState
274
275 // Stream through top-level keys.
276 for dec.More() {
277 tok, err := dec.Token()
278 if err != nil {
279 return nil, fmt.Errorf("token: %w", err)
280 }
281 key, ok := tok.(string)
282 if !ok {
283 continue
284 }
285
286 switch key {
287 case "nodes":
288 // Read opening '[' of nodes array.
289 if _, err := dec.Token(); err != nil {
290 return nil, fmt.Errorf("nodes array open: %w", err)
291 }
292
293 // Phase 1: Decode each NodeRecord one at a time.
294 nodeIdx := 0
295 for dec.More() {
296 var rec NodeRecord
297 if err := dec.Decode(&rec); err != nil {
298 return nil, fmt.Errorf("node %d decode: %w", nodeIdx, err)
299 }
300
301 // Create node with constraints.
302 constraints := make([]axiom.Constraint, len(rec.Tags))
303 for j, tag := range rec.Tags {
304 constraints[j] = cf(tag)
305 }
306 l.AddNode(constraints)
307
308 // Buffer state for phases 2+3.
309 st := nodeState{
310 neighborIDs: rec.Neighbors,
311 lockIn: rec.LockIn,
312 hex: rec.Hex,
313 perm: rec.Perm,
314 projVertex: rec.ProjVertex,
315 projKey: rec.ProjKey,
316 projPath: rec.ProjPath,
317 age: rec.Age,
318 bondCount: rec.BondCount,
319 candidates: rec.Candidates,
320 }
321 if rec.Occupant != nil {
322 st.occType = rec.Occupant.Type
323 st.occValue = rec.Occupant.Value
324 }
325 states = append(states, st)
326
327 nodeIdx++
328 if nodeIdx%1000000 == 0 {
329 log.Printf(" stream-thaw: %dM nodes read", nodeIdx/1000000)
330 }
331 }
332
333 // Read closing ']'.
334 if _, err := dec.Token(); err != nil {
335 return nil, fmt.Errorf("nodes array close: %w", err)
336 }
337
338 default:
339 // Skip non-nodes fields (spore, version, frozen_at).
340 var discard json.RawMessage
341 if err := dec.Decode(&discard); err != nil {
342 return nil, fmt.Errorf("skip %s: %w", key, err)
343 }
344 }
345 }
346
347 // Read closing '}'.
348 dec.Token()
349
350 allNodes := l.Nodes()
351 log.Printf(" stream-thaw: phase 1 complete, %d nodes created", len(allNodes))
352
353 // Phase 2: Connect neighbors + restore state in a single pass.
354 // Edges are bidirectional — connect only when nbID > myID to
355 // deduplicate without any map. O(E) time, O(1) extra space.
356 // Each state entry is zeroed after use to allow progressive GC.
357 for i := range states {
358 st := &states[i]
359 n := allNodes[i]
360 myID := uint64(i)
361
362 // Connect neighbors.
363 for _, nbID := range st.neighborIDs {
364 if nbID > myID && int(nbID) < len(allNodes) {
365 l.Connect(allNodes[myID], allNodes[nbID])
366 }
367 }
368
369 // Restore occupant.
370 if st.occType != "" {
371 elem := frozenElement{typ: st.occType, val: st.occValue}
372 n.ForceOccupant(elem, st.bondCount)
373 }
374
375 // Restore candidates.
376 for _, c := range st.candidates {
377 elem := frozenElement{typ: c.Type, val: c.Value}
378 n.AddCandidate(elem)
379 }
380
381 // Restore projection, hex, lock-in.
382 n.RestoreAge(st.age)
383 n.SetPermutation(st.perm)
384 n.SetProjection(st.projVertex, st.projKey, st.projPath)
385
386 hex := state.Hexagram(st.hex)
387 n.SetEnergy(hex.Inner().Energy())
388 n.SetOuterTrigram(hex.Outer())
389
390 n.RestoreLockIn(st.lockIn)
391
392 // Zero entry to allow GC of strings and slices.
393 states[i] = nodeState{}
394
395 if (i+1)%1000000 == 0 {
396 log.Printf(" stream-thaw: %dM nodes connected+restored", (i+1)/1000000)
397 }
398 }
399 states = nil // release entire buffer
400 log.Printf(" stream-thaw: phase 2 complete, neighbors connected + state restored")
401
402 return l, nil
403 }
404