main.go raw
1 // Command curiosity implements the autonomous curiosity loop.
2 //
3 // After corpus training, the morpheme lattice walks its own weak regions
4 // to produce fragmentary "baby talk" queries, searches archive.org and
5 // Gutenberg for matching texts, fetches one text at a time, feeds it
6 // through the morpheme enzyme, and evaluates whether it helped. When
7 // nothing new bonds, it stops.
8 //
9 // All queries and ingested texts are recorded in curiosity_journal.log
10 // for post-run review.
11 package main
12
13 import (
14 "context"
15 "encoding/json"
16 "flag"
17 "fmt"
18 "io"
19 "log"
20 "net/http"
21 "os"
22 "os/signal"
23 "strings"
24 "time"
25
26 "crypto/rand"
27 "encoding/binary"
28
29 "git.mleku.dev/mleku/dendrite/pkg/axiom"
30 "git.mleku.dev/mleku/dendrite/pkg/forage"
31 "git.mleku.dev/mleku/dendrite/pkg/gap"
32 "git.mleku.dev/mleku/dendrite/pkg/grammar"
33 "git.mleku.dev/mleku/dendrite/pkg/grow"
34 "git.mleku.dev/mleku/dendrite/pkg/lattice"
35 "git.mleku.dev/mleku/dendrite/pkg/mindsicle"
36 "git.mleku.dev/mleku/dendrite/pkg/reflex"
37 )
38
39 // curiosityCheckpoint records loop state for resumption.
40 type curiosityCheckpoint struct {
41 Round int `json:"round"`
42 SeenURLs map[string]string `json:"seen_urls"` // url → title
43 TotalBonded int `json:"total_bonded"`
44 BondRateEWMA float64 `json:"bond_rate_ewma"`
45 ZeroStreak int `json:"zero_streak"`
46 Seed uint64 `json:"seed"`
47 SavedAt string `json:"saved_at"`
48 }
49
50 func main() {
51 var (
52 latticePath = flag.String("lattice", ".condense_morpheme_fast_db.mindsicle.bin", "path to morpheme mindsicle")
53 arcsPath = flag.String("arcs", ".condense_rune_db.arcs.bin", "path to reflex arcs binary")
54 walkLen = flag.Int("walk-len", 5000, "walk steps per gap probe")
55 maxRounds = flag.Int("max-rounds", 0, "0 = infinite until convergence")
56 zeroLimit = flag.Int("zero-streak", 5, "stop after N consecutive zero-bond texts")
57 fetchRate = flag.Duration("rate", 3*time.Second, "delay between fetches")
58 bandwidth = flag.Int64("bandwidth", 0, "total session bandwidth cap in bytes (0=unlimited)")
59 seed = flag.Uint64("seed", 0, "random seed (0=time-based)")
60 outputDB = flag.String("output", ".curiosity_db", "checkpoint file prefix")
61 journalPath = flag.String("journal", "curiosity_journal.log", "audit log path")
62 )
63 flag.Parse()
64
65 if *seed == 0 {
66 *seed = uint64(time.Now().UnixNano())
67 }
68
69 ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
70 defer cancel()
71
72 // --- Load reflex arcs ---
73 log.Printf("curiosity: loading arcs from %s", *arcsPath)
74 arcFile, err := os.Open(*arcsPath)
75 if err != nil {
76 log.Fatalf("curiosity: cannot open arcs: %v", err)
77 }
78 arcs, err := reflex.Load(arcFile)
79 arcFile.Close()
80 if err != nil {
81 log.Fatalf("curiosity: cannot load arcs: %v", err)
82 }
83 log.Printf("curiosity: loaded %d arcs", len(arcs))
84
85 arcIdx := reflex.BuildArcIndex(arcs)
86 enzyme := reflex.NewMorphemeEnzyme(arcIdx)
87
88 cf := func(tag string) axiom.Constraint {
89 return grammar.NewConstraint(tag, grammar.MorphemeText)
90 }
91
92 // --- Load lattice ---
93 log.Printf("curiosity: loading lattice from %s", *latticePath)
94 f, err := os.Open(*latticePath)
95 if err != nil {
96 log.Fatalf("curiosity: cannot open lattice: %v", err)
97 }
98 lat, _, err := mindsicle.StreamThaw(f, cf)
99 f.Close()
100 if err != nil {
101 log.Fatalf("curiosity: cannot thaw lattice: %v", err)
102 }
103
104 h := lat.Health()
105 log.Printf("curiosity: lattice loaded: %d/%d occupied (%.1f%%)",
106 h.Occupied, h.NodeCount, h.OccupancyRate.Float64()*100)
107
108 // Build walk columns for cache-friendly walks.
109 lat.BuildAndAttachWalkColumns()
110 wc := lat.GetWalkColumns()
111 log.Printf("curiosity: walk columns built (%d nodes)", lat.Size())
112
113 // --- Load or create checkpoint ---
114 ckptPath := *outputDB + ".checkpoint.json"
115 var ckpt curiosityCheckpoint
116 if data, err := os.ReadFile(ckptPath); err == nil {
117 if err := json.Unmarshal(data, &ckpt); err == nil {
118 log.Printf("curiosity: resumed from round %d (%d URLs seen, streak=%d)",
119 ckpt.Round, len(ckpt.SeenURLs), ckpt.ZeroStreak)
120 *seed = ckpt.Seed
121 }
122 }
123 if ckpt.SeenURLs == nil {
124 ckpt.SeenURLs = make(map[string]string)
125 }
126 ckpt.Seed = *seed
127
128 // --- Open journal ---
129 journal, err := os.OpenFile(*journalPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
130 if err != nil {
131 log.Fatalf("curiosity: cannot open journal: %v", err)
132 }
133 defer journal.Close()
134 fmt.Fprintf(journal, "# curiosity session started at %s (seed=%d)\n",
135 time.Now().Format(time.RFC3339), *seed)
136
137 // --- Bandwidth tracker ---
138 var totalBytesUsed int64
139
140 // --- Main loop ---
141 startRound := ckpt.Round + 1
142 if *bandwidth > 0 {
143 log.Printf("curiosity: starting from round %d (walk=%d, zero-limit=%d, bandwidth=%dMB)",
144 startRound, *walkLen, *zeroLimit, *bandwidth/(1<<20))
145 } else {
146 log.Printf("curiosity: starting from round %d (walk=%d, zero-limit=%d, bandwidth=unlimited)",
147 startRound, *walkLen, *zeroLimit)
148 }
149
150 for round := startRound; ; round++ {
151 if *maxRounds > 0 && round > *maxRounds {
152 log.Printf("curiosity: max rounds reached (%d)", *maxRounds)
153 break
154 }
155
156 select {
157 case <-ctx.Done():
158 log.Printf("curiosity: interrupted at round %d, saving", round)
159 saveCheckpoint(ckptPath, ckpt)
160 saveLattice(lat, *latticePath)
161 return
162 default:
163 }
164
165 // 1. Detect gaps.
166 gaps := gap.DetectGaps(lat, nil, nil)
167
168 // 2. Walk weak regions.
169 walkSeed := *seed + uint64(round)*0xbeef
170 walkedValues := forage.WeakRegionWalk(lat, wc, *walkLen, walkSeed)
171
172 // 3. Extract baby talk query.
173 query := forage.ExtractBabyTalk(walkedValues)
174 if query == "" && len(gaps) > 0 {
175 // Use gap description as fallback query.
176 query = gaps[0].Description
177 }
178
179 // 4. Search.
180 var pick *forage.SearchResult
181 if query != "" {
182 log.Printf("curiosity: round %d: query=%q", round, truncate(query, 80))
183 results := forage.SearchAll(ctx, query)
184
185 // Convert seenURLs to the format PickBestResult expects.
186 seenSet := make(map[string]bool, len(ckpt.SeenURLs))
187 for u := range ckpt.SeenURLs {
188 seenSet[u] = true
189 }
190 pick = forage.PickBestResult(results, seenSet)
191 }
192
193 // 5. Fallback to random Gutenberg.
194 if pick == nil {
195 r := forage.RandomGutenbergURL(walkSeed)
196 seenSet := make(map[string]bool, len(ckpt.SeenURLs))
197 for u := range ckpt.SeenURLs {
198 seenSet[u] = true
199 }
200 if !seenSet[r.URL] {
201 pick = &r
202 }
203 if pick != nil {
204 log.Printf("curiosity: round %d: fallback to %s", round, pick.Title)
205 }
206 }
207
208 if pick == nil {
209 log.Printf("curiosity: round %d: no unseen texts available, stopping", round)
210 break
211 }
212
213 // 6. Fetch the text.
214 if *bandwidth > 0 && totalBytesUsed >= *bandwidth {
215 log.Printf("curiosity: bandwidth cap reached (%d bytes), stopping", totalBytesUsed)
216 break
217 }
218
219 var fetchLimit int64
220 if *bandwidth > 0 {
221 fetchLimit = *bandwidth - totalBytesUsed
222 } else {
223 fetchLimit = 50 << 20 // 50MB per-fetch safety cap
224 }
225 text, size, err := fetchText(ctx, pick.URL, fetchLimit)
226 if err != nil {
227 log.Printf("curiosity: round %d: fetch failed: %v", round, err)
228 ckpt.SeenURLs[pick.URL] = pick.Title // mark as seen to skip next time
229 ckpt.ZeroStreak++
230 continue
231 }
232 totalBytesUsed += int64(size)
233
234 log.Printf("curiosity: -> %s %q (%s) %s", pick.Source, truncate(pick.Title, 50), formatBytes(size), pick.URL)
235
236 // 7. Digest through morpheme enzyme and feed into lattice.
237 elems := enzyme.DigestText(text)
238 bonded := feedElements(ctx, lat, elems)
239
240 // 8. Dissolve weak bonds.
241 dissolved := dissolveWeak(lat)
242
243 // 9. Update health and convergence.
244 h = lat.Health()
245 bondRate := 0.0
246 if len(elems) > 0 {
247 bondRate = float64(bonded) / float64(len(elems))
248 }
249 ckpt.BondRateEWMA = 0.8*ckpt.BondRateEWMA + 0.2*bondRate
250 ckpt.TotalBonded += bonded
251 ckpt.Round = round
252 ckpt.SeenURLs[pick.URL] = pick.Title
253
254 if bonded == 0 {
255 ckpt.ZeroStreak++
256 } else {
257 ckpt.ZeroStreak = 0
258 }
259
260 // 10. Log to terminal and journal.
261 log.Printf("curiosity: digest: %d morphemes, bonded: %d (%.1f%%), dissolved: %d",
262 len(elems), bonded, bondRate*100, dissolved)
263 log.Printf("curiosity: health: %dK/%dK (%.1f%%) streak: %d ewma: %.4f",
264 h.Occupied/1000, h.NodeCount/1000, h.OccupancyRate.Float64()*100,
265 ckpt.ZeroStreak, ckpt.BondRateEWMA)
266
267 fmt.Fprintf(journal, "%s round=%d query=%q source=%s title=%q url=%s size=%s morphemes=%d bonded=%d rate=%.1f%% health=%.1f%% streak=%d\n",
268 time.Now().Format(time.RFC3339), round,
269 truncate(query, 60), pick.Source, pick.Title, pick.URL,
270 formatBytes(size), len(elems), bonded, bondRate*100,
271 h.OccupancyRate.Float64()*100, ckpt.ZeroStreak)
272
273 // 11. Checkpoint every 5 rounds.
274 if round%5 == 0 {
275 saveCheckpoint(ckptPath, ckpt)
276 saveLattice(lat, *latticePath)
277 log.Printf("curiosity: checkpoint saved at round %d", round)
278 }
279
280 // 12. Dynamic growth: expand lattice when occupancy > 80%.
281 if h.OccupancyRate.Float64() > 0.80 {
282 growBy := h.NodeCount / 2 // grow by 50% of current size
283 var expandSeed [32]byte
284 binary.LittleEndian.PutUint64(expandSeed[:], *seed+uint64(round)*0xdead)
285 rand.Read(expandSeed[8:]) // mix in true randomness
286 added := grammar.ExpandLattice(lat, grammar.MorphemeText, growBy, expandSeed, cf, grammar.MorphemeDefaultCounts)
287 if added > 0 {
288 // Rebuild walk columns with new nodes.
289 lat.BuildAndAttachWalkColumns()
290 wc = lat.GetWalkColumns()
291 log.Printf("curiosity: expanded lattice by %d nodes (now %d)", added, lat.Size())
292 // Reset zero streak — new capacity means new opportunities.
293 ckpt.ZeroStreak = 0
294 }
295 }
296
297 // 13. Check convergence: only zero-streak stops the loop now.
298 if ckpt.ZeroStreak >= *zeroLimit {
299 log.Printf("curiosity: converged — %d consecutive zero-bond texts", ckpt.ZeroStreak)
300 break
301 }
302
303 // Rate limit.
304 select {
305 case <-time.After(*fetchRate):
306 case <-ctx.Done():
307 }
308 }
309
310 // Final save.
311 saveCheckpoint(ckptPath, ckpt)
312 saveLattice(lat, *latticePath)
313
314 h = lat.Health()
315 fmt.Fprintf(journal, "# curiosity session ended at %s: %d rounds, %d total bonded, %dK/%dK occupied (%.1f%%)\n",
316 time.Now().Format(time.RFC3339), ckpt.Round, ckpt.TotalBonded,
317 h.Occupied/1000, h.NodeCount/1000, h.OccupancyRate.Float64()*100)
318
319 log.Printf("curiosity: done — %d rounds, %d total bonded, %d URLs explored",
320 ckpt.Round, ckpt.TotalBonded, len(ckpt.SeenURLs))
321 }
322
323 // fetchText downloads a plain text file from a URL, respecting a byte limit.
324 func fetchText(ctx context.Context, rawURL string, maxBytes int64) (string, int, error) {
325 req, err := http.NewRequestWithContext(ctx, "GET", rawURL, nil)
326 if err != nil {
327 return "", 0, err
328 }
329 req.Header.Set("User-Agent", "dendrite-curiosity/0.1")
330
331 resp, err := (&http.Client{Timeout: 30 * time.Second}).Do(req)
332 if err != nil {
333 return "", 0, err
334 }
335 defer resp.Body.Close()
336
337 if resp.StatusCode != 200 {
338 return "", 0, fmt.Errorf("status %d", resp.StatusCode)
339 }
340
341 body, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes))
342 if err != nil {
343 return "", 0, err
344 }
345
346 return string(body), len(body), nil
347 }
348
349 // feedElements feeds elements into the lattice via grow.Run.
350 func feedElements(ctx context.Context, lat *lattice.Lattice, elems []axiom.Element) int {
351 if len(elems) == 0 {
352 return 0
353 }
354
355 ch := make(chan axiom.Element, len(elems))
356 go func() {
357 for _, e := range elems {
358 ch <- e
359 }
360 close(ch)
361 }()
362
363 cfg := grow.Config{
364 MaxSteps: 10,
365 Workers: grow.WorkerCount(),
366 }
367
368 events := make(chan grow.Event, 256)
369 bonded := 0
370 done := make(chan struct{})
371 go func() {
372 for ev := range events {
373 if ev.Type == grow.EventBonded {
374 bonded++
375 }
376 }
377 close(done)
378 }()
379
380 grow.Run(ctx, lat, ch, cfg, events)
381 close(events)
382 <-done
383 return bonded
384 }
385
386 // dissolveWeak removes occupants with low contextual lock-in.
387 func dissolveWeak(l *lattice.Lattice) int {
388 nodes := l.Nodes()
389 target := max(1, len(nodes)/100)
390
391 type scored struct {
392 node *lattice.Node
393 score float64
394 }
395 var weak []scored
396 for _, n := range nodes {
397 if !n.Occupied() {
398 continue
399 }
400 s := n.ContextualLockIn().Float64()
401 if s < 0.3 {
402 weak = append(weak, scored{n, s})
403 }
404 }
405
406 dissolved := 0
407 for _, s := range weak {
408 if dissolved >= target {
409 break
410 }
411 s.node.Dissolve()
412 l.ReindexVacant(s.node)
413 dissolved++
414 }
415 return dissolved
416 }
417
418 // saveCheckpoint writes the curiosity state for resumption.
419 func saveCheckpoint(path string, ckpt curiosityCheckpoint) {
420 ckpt.SavedAt = time.Now().Format(time.RFC3339)
421 data, err := json.MarshalIndent(ckpt, "", " ")
422 if err != nil {
423 log.Printf("curiosity: WARNING: cannot marshal checkpoint: %v", err)
424 return
425 }
426 tmp := path + ".tmp"
427 if err := os.WriteFile(tmp, data, 0o644); err != nil {
428 log.Printf("curiosity: WARNING: cannot write checkpoint: %v", err)
429 return
430 }
431 os.Rename(tmp, path)
432 }
433
434 // saveLattice writes the lattice atomically.
435 func saveLattice(lat *lattice.Lattice, path string) {
436 tmp := path + ".tmp"
437 f, err := os.Create(tmp)
438 if err != nil {
439 log.Printf("curiosity: WARNING: cannot create %s: %v", tmp, err)
440 return
441 }
442 if err := mindsicle.StreamFreeze(lat, nil, f); err != nil {
443 f.Close()
444 os.Remove(tmp)
445 log.Printf("curiosity: WARNING: save failed: %v", err)
446 return
447 }
448 f.Close()
449 if err := os.Rename(tmp, path); err != nil {
450 log.Printf("curiosity: WARNING: rename failed: %v", err)
451 return
452 }
453 fi, _ := os.Stat(path)
454 if fi != nil {
455 log.Printf("curiosity: saved %.2f MB", float64(fi.Size())/(1024*1024))
456 }
457 }
458
459 func truncate(s string, n int) string {
460 s = strings.ReplaceAll(s, "\n", " ")
461 if len(s) > n {
462 return s[:n] + "..."
463 }
464 return s
465 }
466
467 func formatBytes(n int) string {
468 switch {
469 case n >= 1<<20:
470 return fmt.Sprintf("%.1fMB", float64(n)/float64(1<<20))
471 case n >= 1<<10:
472 return fmt.Sprintf("%.1fKB", float64(n)/float64(1<<10))
473 default:
474 return fmt.Sprintf("%dB", n)
475 }
476 }
477