main.go raw
1 // Command recognise builds and queries text recognition lattices.
2 //
3 // Modes:
4 //
5 // recognise -train -corpus ./texts -memory ./recog_db
6 // recognise -detect -memory ./recog_db -sample ./input.txt
7 // recognise -fingerprint -model chatgpt -corpus ./chatgpt_texts -memory ./recog_db
8 package main
9
10 import (
11 "context"
12 "encoding/json"
13 "flag"
14 "fmt"
15 "log"
16 "os"
17 "os/signal"
18 "path/filepath"
19 "strings"
20
21 "git.mleku.dev/mleku/dendrite/pkg/axiom"
22 "git.mleku.dev/mleku/dendrite/pkg/converge"
23 "git.mleku.dev/mleku/dendrite/pkg/enzyme"
24 "git.mleku.dev/mleku/dendrite/pkg/extract"
25 "git.mleku.dev/mleku/dendrite/pkg/grammar"
26 "git.mleku.dev/mleku/dendrite/pkg/grow"
27 "git.mleku.dev/mleku/dendrite/pkg/lattice"
28 "git.mleku.dev/mleku/dendrite/pkg/memory"
29 "git.mleku.dev/mleku/dendrite/pkg/mindsicle"
30 "git.mleku.dev/mleku/dendrite/pkg/profile"
31 )
32
33 func main() {
34 var (
35 train = flag.Bool("train", false, "train baseline lattice from corpus directory")
36 detect = flag.Bool("detect", false, "detect AI text in a sample")
37 fingerprint = flag.Bool("fingerprint", false, "build model fingerprint from labeled corpus")
38
39 corpusDir = flag.String("corpus", "", "corpus directory (for -train and -fingerprint)")
40 memoryDir = flag.String("memory", ".recognise_db", "persistent memory directory")
41 trollMemory = flag.String("troll-memory", "", "badger DB with manipulation-trained mindsicles (for -detect)")
42 trollPassN = flag.Int("troll-passes", 8, "number of troll detection passes")
43 sampleFile = flag.String("sample", "", "text file to analyze (for -detect)")
44 modelName = flag.String("model", "", "model name (for -fingerprint)")
45 latticeN = flag.Int("nodes", 4096, "number of lattice nodes")
46 maxSteps = flag.Int("max-steps", 500, "max walk steps per element")
47 passes = flag.Int("passes", 4, "number of recognition passes (1=word only, 2+=event levels)")
48 window = flag.Int("window", 500, "max tokens to process per sample (0=unlimited)")
49 )
50
51 flag.Parse()
52
53 ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
54 defer cancel()
55
56 modeCount := 0
57 if *train {
58 modeCount++
59 }
60 if *detect {
61 modeCount++
62 }
63 if *fingerprint {
64 modeCount++
65 }
66 if modeCount != 1 {
67 fmt.Fprintln(os.Stderr, "exactly one of -train, -detect, -fingerprint required")
68 flag.Usage()
69 os.Exit(1)
70 }
71
72 switch {
73 case *train:
74 if *corpusDir == "" {
75 log.Fatal("-corpus required for -train mode")
76 }
77 if err := runTrain(ctx, *corpusDir, *memoryDir, *latticeN, *maxSteps, *passes); err != nil {
78 log.Fatal(err)
79 }
80
81 case *detect:
82 if *sampleFile == "" {
83 log.Fatal("-sample required for -detect mode")
84 }
85 if err := runDetect(ctx, *sampleFile, *memoryDir, trollMemory, *latticeN, *maxSteps, *passes, *trollPassN, *window); err != nil {
86 log.Fatal(err)
87 }
88
89 case *fingerprint:
90 if *corpusDir == "" || *modelName == "" {
91 log.Fatal("-corpus and -model required for -fingerprint mode")
92 }
93 if err := runFingerprint(ctx, *corpusDir, *memoryDir, *modelName, *latticeN, *maxSteps); err != nil {
94 log.Fatal(err)
95 }
96 }
97 }
98
99 // runTrain builds an N-pass recognition system from a human text corpus.
100 //
101 // Pass 1: Train a word-level lattice from the corpus until convergence.
102 // Pass 2..N: Each subsequent pass takes the bond events from the previous
103 // pass, classifies them (snap/near/far/miss), and grows them into a new
104 // event-level lattice. Each level captures progressively higher-order
105 // structural patterns — the rhythm of rhythms.
106 //
107 // All lattices are saved as mindsicles (gen 1..N) for detection.
108 func runTrain(ctx context.Context, corpusDir, memoryDir string, latticeSize, maxSteps, numPasses int) error {
109 if numPasses < 1 {
110 numPasses = 1
111 }
112
113 db, err := memory.Open(memoryDir)
114 if err != nil {
115 return fmt.Errorf("open memory: %w", err)
116 }
117 defer db.Close()
118
119 l, seed := buildTextLattice(latticeSize)
120 tracker := converge.NewTracker(converge.DefaultWindowSize, converge.DefaultThreshold)
121
122 cfg := grow.Config{
123 MaxSteps: maxSteps,
124 Workers: grow.WorkerCount(),
125 }
126
127 registry := extract.NewRegistry()
128 files, err := collectFiles(corpusDir)
129 if err != nil {
130 return fmt.Errorf("walk corpus: %w", err)
131 }
132
133 fmt.Printf("=== pass 1/%d: word-level lattice ===\n", numPasses)
134 fmt.Printf("training on %d files from %s\n", len(files), corpusDir)
135 fmt.Printf("lattice: %d nodes, seed: %x...\n", l.Size(), seed[:4])
136
137 // Pass 1: Grow word lattice until convergence.
138 convergedAt := len(files)
139 for i, path := range files {
140 select {
141 case <-ctx.Done():
142 fmt.Println("\ninterrupted, saving progress...")
143 convergedAt = i
144 goto chain
145 default:
146 }
147
148 if err := ingestFileSimple(ctx, l, path, registry, cfg, tracker); err != nil {
149 fmt.Fprintf(os.Stderr, " skip %s: %v\n", path, err)
150 continue
151 }
152
153 if (i+1)%100 == 0 || i == len(files)-1 {
154 h := l.Health()
155 fmt.Printf(" [%d/%d] occupied=%d/%d convergence_rate=%s\n",
156 i+1, len(files), h.Occupied, h.NodeCount,
157 tracker.Report().CurrentRate.String())
158 }
159
160 if tracker.IsConverged() {
161 convergedAt = i + 1
162 fmt.Printf(" converged at file %d/%d\n", convergedAt, len(files))
163 break
164 }
165 }
166
167 chain:
168 // Save word lattice as gen 1.
169 if err := saveLattice(db, l, 1); err != nil {
170 return err
171 }
172
173 h := l.Health()
174 fmt.Printf("word lattice: %d/%d occupied\n", h.Occupied, h.NodeCount)
175
176 // Validation files for event-level passes.
177 // Cap validation to 10 files — more doesn't help and takes forever.
178 validationFiles := files[convergedAt:]
179 if len(validationFiles) == 0 {
180 n := len(files)
181 if n > 5 {
182 n = 5
183 }
184 validationFiles = files[:n]
185 }
186 if len(validationFiles) > 10 {
187 validationFiles = validationFiles[:10]
188 }
189
190 if numPasses < 2 {
191 return nil
192 }
193
194 // Build the chain of event lattices (passes 2..N).
195 // Each pass: clear previous lattice, run validation through it,
196 // collect events, classify, grow into next event lattice.
197 eventLattices := make([]*lattice.Lattice, numPasses-1)
198 for i := range eventLattices {
199 // Each successive event lattice is smaller — fewer patterns at higher levels.
200 sz := latticeSize / (2 * (i + 1))
201 if sz < 64 {
202 sz = 64
203 }
204 eventLattices[i] = buildEventLattice(sz)
205 }
206
207 // Run the chain. For each pass p (2..N):
208 // - Source lattice = pass p-1's lattice (cleared)
209 // - Target lattice = pass p's event lattice
210 // - Input = validation files (pass 2) or previous events (pass 3+)
211
212 // First, collect word-level events from validation files.
213 l.ClearOccupants()
214
215 fmt.Printf("\n=== pass 2/%d: event-level lattice ===\n", numPasses)
216 fmt.Printf("validation files: %d\n", len(validationFiles))
217
218 // Collect all word-level events from validation files.
219 var allWordEvents []grow.Event
220 for _, path := range validationFiles {
221 select {
222 case <-ctx.Done():
223 goto done
224 default:
225 }
226 evs, err := collectFileEvents(ctx, l, path, registry, cfg)
227 if err != nil {
228 fmt.Fprintf(os.Stderr, " skip %s: %v\n", path, err)
229 continue
230 }
231 allWordEvents = append(allWordEvents, evs...)
232 }
233
234 // Now chain through passes 2..N.
235 {
236 prevEvents := allWordEvents
237 for p := 0; p < len(eventLattices); p++ {
238 passNum := p + 2
239 el := eventLattices[p]
240
241 // Classify previous events and grow into this event lattice.
242 classified := classifyEvents(prevEvents)
243 nextEvents := growElements(ctx, el, classified, cfg, false)
244
245 bonded, expired, total := countEvents(nextEvents)
246 fmt.Printf(" pass %d/%d: bonded=%d expired=%d total=%d rate=%.2f%%\n",
247 passNum, numPasses, bonded, expired, total, pct(bonded, total))
248
249 // Save this event lattice.
250 if err := saveLattice(db, el, uint32(passNum)); err != nil {
251 return err
252 }
253
254 eh := el.Health()
255 fmt.Printf(" event lattice %d: %d/%d occupied\n", passNum, eh.Occupied, eh.NodeCount)
256
257 if p < len(eventLattices)-1 {
258 fmt.Printf("\n=== pass %d/%d: meta-event lattice ===\n", passNum+1, numPasses)
259 // Clear this event lattice and use it as source for next pass.
260 el.ClearOccupants()
261 prevEvents = nextEvents
262 }
263 }
264 }
265
266 done:
267 fmt.Printf("\n=== training complete: %d passes ===\n", numPasses)
268 return nil
269 }
270
271 // runDetect tests whether sample text bonds through the N-pass lattice chain.
272 //
273 // Each pass thaws its trained lattice, clears occupants, and grows input
274 // through it. The bond events from each pass become the input for the next.
275 // All lattices are disposable copies — the trained state is never modified.
276 func runDetect(ctx context.Context, sampleFile, memoryDir string, trollMemoryDir *string, _, maxSteps, numPasses, trollPasses, tokenWindow int) error {
277 if numPasses < 1 {
278 numPasses = 1
279 }
280
281 db, err := memory.Open(memoryDir)
282 if err != nil {
283 return fmt.Errorf("open memory: %w", err)
284 }
285 defer db.Close()
286
287 probeCfg := grow.Config{
288 MaxSteps: 3,
289 Workers: 1,
290 }
291
292 // Thaw AI detection lattices.
293 lattices, err := thawLattices(db, numPasses, "ai")
294 if err != nil {
295 return err
296 }
297
298 // Print AI lattice info.
299 fmt.Println("=== AI detection lattices ===")
300 printLatticeInfo(lattices)
301
302 // Thaw troll detection lattices if configured.
303 var trollLattices []*lattice.Lattice
304 if *trollMemoryDir != "" {
305 trollDB, err := memory.Open(*trollMemoryDir)
306 if err != nil {
307 return fmt.Errorf("open troll memory: %w", err)
308 }
309 trollLattices, err = thawLattices(trollDB, trollPasses, "troll")
310 trollDB.Close()
311 if err != nil {
312 return err
313 }
314 fmt.Println("\n=== manipulation detection lattices ===")
315 printLatticeInfo(trollLattices)
316 }
317
318 // Read and tokenize sample.
319 f, err := os.Open(sampleFile)
320 if err != nil {
321 return err
322 }
323 defer f.Close()
324
325 rawSolution := enzyme.Text{}.Digest(f)
326 solution := limitTokens(rawSolution, tokenWindow)
327
328 // Materialize tokens so both chains can use them.
329 var tokens []axiom.Element
330 for tok := range solution {
331 tokens = append(tokens, tok)
332 }
333
334 fmt.Printf("\nsample: %s (%d tokens)\n", sampleFile, len(tokens))
335
336 // --- AI chain ---
337 fmt.Println("\n=== AI detection ===")
338 aiStats := runChain(ctx, lattices, numPasses, tokens, probeCfg, true)
339
340 // --- Troll chain ---
341 var trollStats []grammar.PassStats
342 if len(trollLattices) > 0 {
343 fmt.Println("\n=== manipulation detection ===")
344 trollStats = runChain(ctx, trollLattices, trollPasses, tokens, probeCfg, true)
345 }
346
347 // Final verdict.
348 verdict := grammar.Score(aiStats)
349 if len(trollStats) > 0 {
350 grammar.ScoreTroll(&verdict, trollStats)
351 }
352 fmt.Println()
353 fmt.Print(verdict.String())
354
355 return nil
356 }
357
358 // thawLattices loads and thaws a chain of mindsicles from a memory DB.
359 func thawLattices(db *memory.DB, passes int, label string) ([]*lattice.Lattice, error) {
360 lats := make([]*lattice.Lattice, passes)
361
362 wordData, err := db.LoadMindsicle(1)
363 if err != nil {
364 return nil, fmt.Errorf("load %s word mindsicle: %w", label, err)
365 }
366 var wm mindsicle.Mindsicle
367 if err := json.Unmarshal(wordData, &wm); err != nil {
368 return nil, fmt.Errorf("unmarshal %s word mindsicle: %w", label, err)
369 }
370 lats[0] = wm.Thaw(func(tag string) axiom.Constraint {
371 return grammar.NewConstraint(tag, grammar.NaturalText)
372 })
373
374 for p := 1; p < passes; p++ {
375 data, err := db.LoadMindsicle(uint32(p + 1))
376 if err != nil {
377 return nil, fmt.Errorf("load %s event mindsicle gen %d: %w", label, p+1, err)
378 }
379 var em mindsicle.Mindsicle
380 if err := json.Unmarshal(data, &em); err != nil {
381 return nil, fmt.Errorf("unmarshal %s event mindsicle gen %d: %w", label, p+1, err)
382 }
383 lats[p] = em.Thaw(func(tag string) axiom.Constraint {
384 return grammar.NewConstraint(tag, grammar.BondEvent)
385 })
386 }
387
388 return lats, nil
389 }
390
391 // printLatticeInfo prints node/occupancy info for a lattice chain.
392 func printLatticeInfo(lats []*lattice.Lattice) {
393 for i, lat := range lats {
394 h := lat.Health()
395 label := "word"
396 if i > 0 {
397 label = fmt.Sprintf("event-L%d", i)
398 }
399 fmt.Printf(" %s: %d nodes, %d/%d occupied\n", label, h.NodeCount, h.Occupied, h.NodeCount)
400 }
401 }
402
403 // runChain runs the full N-pass SeqProbe chain and returns pass stats.
404 // If verbose is true, prints per-pass details to stdout.
405 func runChain(ctx context.Context, lats []*lattice.Lattice, passes int, tokens []axiom.Element, probeCfg grow.Config, verbose bool) []grammar.PassStats {
406 // Pass 1: probe tokens against word lattice.
407 tokenCh := make(chan axiom.Element, len(tokens))
408 for _, tok := range tokens {
409 tokenCh <- tok
410 }
411 close(tokenCh)
412
413 eventsCh := make(chan grow.Event, 256)
414 var prevEvents []grow.Event
415 done := make(chan struct{})
416 go func() {
417 for ev := range eventsCh {
418 prevEvents = append(prevEvents, ev)
419 }
420 close(done)
421 }()
422 grow.SeqProbe(ctx, lats[0], tokenCh, probeCfg, eventsCh)
423 close(eventsCh)
424 <-done
425
426 allStats := make([]grammar.PassStats, passes)
427 b, e, t := countEvents(prevEvents)
428 allStats[0] = grammar.PassStats{Events: prevEvents, Bonded: b, Expired: e, Total: t}
429
430 if verbose {
431 fmt.Printf("\n pass 1/%d (word):\n", passes)
432 fmt.Printf(" tokens: %d\n", t)
433 fmt.Printf(" bonded: %d (%.2f%%)\n", b, pct(b, t))
434 fmt.Printf(" expired: %d (%.2f%%)\n", e, pct(e, t))
435 printEventDist(prevEvents, t)
436 printWalkStats(prevEvents)
437 }
438
439 // Chain passes 2..N.
440 for p := 1; p < passes; p++ {
441 classified := classifyEvents(prevEvents)
442 nextEvents := seqProbeElements(ctx, lats[p], classified, probeCfg)
443
444 b, e, t = countEvents(nextEvents)
445 allStats[p] = grammar.PassStats{Events: nextEvents, Bonded: b, Expired: e, Total: t}
446
447 if verbose {
448 fmt.Printf("\n pass %d/%d (event-L%d):\n", p+1, passes, p)
449 fmt.Printf(" events: %d\n", t)
450 fmt.Printf(" bonded: %d (%.2f%%)\n", b, pct(b, t))
451 fmt.Printf(" expired: %d (%.2f%%)\n", e, pct(e, t))
452 printEventDist(nextEvents, t)
453 printWalkStats(nextEvents)
454 }
455
456 prevEvents = nextEvents
457 }
458
459 return allStats
460 }
461
462 // runFingerprint builds a model-specific lattice from labeled corpus.
463 func runFingerprint(ctx context.Context, corpusDir, memoryDir, modelName string, latticeSize, maxSteps int) error {
464 db, err := memory.Open(memoryDir)
465 if err != nil {
466 return fmt.Errorf("open memory: %w", err)
467 }
468 defer db.Close()
469
470 l, _ := buildTextLattice(latticeSize)
471 collector := profile.NewCollector()
472 tracker := converge.NewTracker(converge.DefaultWindowSize, converge.DefaultThreshold)
473
474 cfg := grow.Config{
475 MaxSteps: maxSteps,
476 Workers: grow.WorkerCount(),
477 }
478
479 registry := extract.NewRegistry()
480 files, err := collectFiles(corpusDir)
481 if err != nil {
482 return fmt.Errorf("walk corpus: %w", err)
483 }
484
485 fmt.Printf("fingerprinting model %q from %d files\n", modelName, len(files))
486
487 for i, path := range files {
488 select {
489 case <-ctx.Done():
490 fmt.Println("\ninterrupted, saving progress...")
491 goto save
492 default:
493 }
494
495 if err := ingestFile(ctx, l, path, registry, cfg, collector, tracker); err != nil {
496 fmt.Fprintf(os.Stderr, " skip %s: %v\n", path, err)
497 continue
498 }
499
500 if (i+1)%100 == 0 || i == len(files)-1 {
501 fmt.Printf(" [%d/%d]\n", i+1, len(files))
502 }
503 }
504
505 save:
506 snap := collector.Snapshot()
507 stats := profile.Compute(snap, l.Size())
508 profData, err := snap.Marshal()
509 if err != nil {
510 return fmt.Errorf("marshal profile: %w", err)
511 }
512 if err := db.RecordModelSpore(modelName, profData); err != nil {
513 return fmt.Errorf("save model spore: %w", err)
514 }
515
516 printStats(modelName, stats)
517 return nil
518 }
519
520 // buildTextLattice creates a lattice with NaturalText grammar topology.
521 func buildTextLattice(size int) (*lattice.Lattice, [32]byte) {
522 counts := grammar.TextDefaultCounts(size)
523 seed := [32]byte{0xDE, 0xAD, 0xBE, 0xEF} // fixed seed for reproducibility
524 l := grammar.BuildGrammarLattice(
525 grammar.NaturalText,
526 counts,
527 seed,
528 func(tag string) axiom.Constraint {
529 return grammar.NewConstraint(tag, grammar.NaturalText)
530 },
531 )
532 return l, seed
533 }
534
535 // buildEventLattice creates a lattice with BondEvent grammar topology.
536 func buildEventLattice(size int) *lattice.Lattice {
537 counts := grammar.EventDefaultCounts(size)
538 seed := [32]byte{0xE0, 0xE1, 0x70, 0x02}
539 return grammar.BuildGrammarLattice(
540 grammar.BondEvent,
541 counts,
542 seed,
543 func(tag string) axiom.Constraint {
544 return grammar.NewConstraint(tag, grammar.BondEvent)
545 },
546 )
547 }
548
549 // ingestFileSimple grows text through the lattice with convergence tracking
550 // but no profile collection.
551 func ingestFileSimple(
552 ctx context.Context,
553 l *lattice.Lattice,
554 path string,
555 registry *extract.Registry,
556 cfg grow.Config,
557 tracker *converge.Tracker,
558 ) error {
559 rc, err := registry.Extract(path)
560 if err != nil {
561 return err
562 }
563 defer rc.Close()
564
565 solution := enzyme.Text{}.Digest(rc)
566
567 counted := make(chan axiom.Element, 64)
568 go func() {
569 defer close(counted)
570 for elem := range solution {
571 tracker.RecordToken()
572 select {
573 case counted <- elem:
574 case <-ctx.Done():
575 return
576 }
577 }
578 }()
579
580 events := make(chan grow.Event, 256)
581 go func() {
582 for range events {
583 }
584 }()
585
586 grow.Run(ctx, l, counted, cfg, events)
587 close(events)
588
589 return nil
590 }
591
592 // collectFileEvents grows a file through a lattice and returns all events.
593 // Caps at maxTokensPerFile tokens to avoid spending forever on large books.
594 const maxTokensPerFile = 50000
595
596 func collectFileEvents(
597 ctx context.Context,
598 l *lattice.Lattice,
599 path string,
600 registry *extract.Registry,
601 cfg grow.Config,
602 ) ([]grow.Event, error) {
603 rc, err := registry.Extract(path)
604 if err != nil {
605 return nil, err
606 }
607 defer rc.Close()
608
609 solution := limitTokens(enzyme.Text{}.Digest(rc), maxTokensPerFile)
610 eventsCh := make(chan grow.Event, 256)
611 var events []grow.Event
612
613 go func() {
614 for ev := range eventsCh {
615 events = append(events, ev)
616 }
617 }()
618
619 grow.Run(ctx, l, solution, cfg, eventsCh)
620 close(eventsCh)
621
622 return events, nil
623 }
624
625 // classifyEvents converts grow.Events into axiom.Elements for the next pass.
626 func classifyEvents(events []grow.Event) []axiom.Element {
627 elems := make([]axiom.Element, len(events))
628 for i, ev := range events {
629 elems[i] = grammar.ClassifyEvent(ev)
630 }
631 return elems
632 }
633
634 // growElements grows a slice of elements into a lattice, returning all events.
635 // If dryRun is true, bonds are immediately reversed (for detection).
636 func growElements(ctx context.Context, l *lattice.Lattice, elems []axiom.Element, cfg grow.Config, dryRun bool) []grow.Event {
637 solution := make(chan axiom.Element, 256)
638 go func() {
639 defer close(solution)
640 for _, e := range elems {
641 select {
642 case solution <- e:
643 case <-ctx.Done():
644 return
645 }
646 }
647 }()
648
649 eventsCh := make(chan grow.Event, 256)
650 var events []grow.Event
651
652 go func() {
653 for ev := range eventsCh {
654 events = append(events, ev)
655 }
656 }()
657
658 if dryRun {
659 grow.DryRun(ctx, l, solution, cfg, eventsCh)
660 } else {
661 grow.Run(ctx, l, solution, cfg, eventsCh)
662 }
663 close(eventsCh)
664
665 return events
666 }
667
668 // countEvents counts bonded/expired/total from an event slice.
669 func countEvents(events []grow.Event) (bonded, expired, total int64) {
670 for _, ev := range events {
671 switch ev.Type {
672 case grow.EventBonded:
673 bonded++
674 case grow.EventExpired:
675 expired++
676 }
677 total++
678 }
679 return
680 }
681
682 // saveLattice freezes and saves a lattice as a mindsicle at the given generation.
683 func saveLattice(db *memory.DB, l *lattice.Lattice, gen uint32) error {
684 m := mindsicle.Freeze(l, nil)
685 data, err := json.Marshal(m)
686 if err != nil {
687 return fmt.Errorf("marshal mindsicle gen %d: %w", gen, err)
688 }
689 if err := db.RecordMindsicle(gen, data); err != nil {
690 return fmt.Errorf("save mindsicle gen %d: %w", gen, err)
691 }
692 fmt.Printf(" saved lattice gen %d (%d bytes)\n", gen, len(data))
693 return nil
694 }
695
696 // seqProbeElements runs SeqProbe over a slice of elements against a trained lattice.
697 func seqProbeElements(ctx context.Context, l *lattice.Lattice, elems []axiom.Element, cfg grow.Config) []grow.Event {
698 solution := make(chan axiom.Element, 256)
699 go func() {
700 defer close(solution)
701 for _, e := range elems {
702 select {
703 case solution <- e:
704 case <-ctx.Done():
705 return
706 }
707 }
708 }()
709
710 eventsCh := make(chan grow.Event, 256)
711 var events []grow.Event
712
713 go func() {
714 for ev := range eventsCh {
715 events = append(events, ev)
716 }
717 }()
718
719 grow.SeqProbe(ctx, l, solution, cfg, eventsCh)
720 close(eventsCh)
721
722 return events
723 }
724
725 // printWalkStats prints walk distance statistics for bonded events.
726 func printWalkStats(events []grow.Event) {
727 var totalSteps int64
728 var bonded int64
729 var maxSteps int
730 for _, ev := range events {
731 if ev.Type == grow.EventBonded {
732 totalSteps += int64(ev.Steps)
733 bonded++
734 if ev.Steps > maxSteps {
735 maxSteps = ev.Steps
736 }
737 }
738 }
739 if bonded == 0 {
740 fmt.Printf(" walk: no bonded events\n")
741 return
742 }
743 avg := float64(totalSteps) / float64(bonded)
744 fmt.Printf(" walk: avg=%.2f max=%d (lower=better fit)\n", avg, maxSteps)
745 }
746
747 // printEventDist prints the hit/miss distribution by element type.
748 func printEventDist(events []grow.Event, total int64) {
749 counts := make(map[string]int64)
750 for _, ev := range events {
751 e := grammar.ClassifyEvent(ev)
752 counts[e.Type()]++
753 }
754
755 // Aggregate hits and misses.
756 var totalHit, totalMiss int64
757 for tag, n := range counts {
758 if strings.HasSuffix(tag, ".hit") {
759 totalHit += n
760 } else {
761 totalMiss += n
762 }
763 }
764 fmt.Printf(" hit=%d(%.0f%%) miss=%d(%.0f%%)\n",
765 totalHit, pct(totalHit, total), totalMiss, pct(totalMiss, total))
766
767 // Print per-type breakdown if there are misses (interesting case).
768 if totalMiss > 0 {
769 fmt.Printf(" miss types:")
770 for _, tag := range grammar.EventTags() {
771 if strings.HasSuffix(tag, ".miss") && counts[tag] > 0 {
772 fmt.Printf(" %s=%d", tag, counts[tag])
773 }
774 }
775 fmt.Println()
776 }
777 }
778
779 // ingestFile extracts text from a file and feeds it through the lattice
780 // with profile collection and convergence tracking. Used by fingerprint mode.
781 func ingestFile(
782 ctx context.Context,
783 l *lattice.Lattice,
784 path string,
785 registry *extract.Registry,
786 cfg grow.Config,
787 collector *profile.Collector,
788 tracker *converge.Tracker,
789 ) error {
790 rc, err := registry.Extract(path)
791 if err != nil {
792 return err
793 }
794 defer rc.Close()
795
796 solution := enzyme.Text{}.Digest(rc)
797
798 counted := make(chan axiom.Element, 64)
799 go func() {
800 defer close(counted)
801 for elem := range solution {
802 tracker.RecordToken()
803 select {
804 case counted <- elem:
805 case <-ctx.Done():
806 return
807 }
808 }
809 }()
810
811 events := make(chan grow.Event, 256)
812 go func() {
813 for ev := range events {
814 collector.RecordGrowEvent(ev)
815 }
816 }()
817
818 grow.Run(ctx, l, counted, cfg, events)
819 close(events)
820
821 return nil
822 }
823
824 // limitTokens wraps a channel to emit at most maxTokens elements.
825 // If maxTokens <= 0, all elements pass through.
826 func limitTokens(in <-chan axiom.Element, maxTokens int) <-chan axiom.Element {
827 if maxTokens <= 0 {
828 return in
829 }
830 out := make(chan axiom.Element, cap(in))
831 go func() {
832 defer close(out)
833 count := 0
834 for e := range in {
835 if count >= maxTokens {
836 // Drain remaining input to avoid blocking the producer.
837 for range in {
838 }
839 return
840 }
841 out <- e
842 count++
843 }
844 }()
845 return out
846 }
847
848 // pct computes a percentage, returning 0 for zero denominator.
849 func pct(num, denom int64) float64 {
850 if denom == 0 {
851 return 0
852 }
853 return float64(num) / float64(denom) * 100
854 }
855
856 // collectFiles recursively collects file paths from a directory.
857 func collectFiles(dir string) ([]string, error) {
858 var files []string
859 err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
860 if err != nil {
861 return nil // skip errors
862 }
863 if info.IsDir() {
864 // Skip hidden directories.
865 if strings.HasPrefix(info.Name(), ".") && info.Name() != "." {
866 return filepath.SkipDir
867 }
868 return nil
869 }
870 // Skip hidden files and very small files.
871 if strings.HasPrefix(info.Name(), ".") || info.Size() < 100 {
872 return nil
873 }
874 files = append(files, path)
875 return nil
876 })
877 return files, err
878 }
879
880 // printStats prints a Stats struct in a readable format.
881 func printStats(label string, s profile.Stats) {
882 fmt.Printf("%s:\n", label)
883 fmt.Printf(" path_entropy: %s (%.4f)\n", s.PathEntropy.String(), s.PathEntropy.Float64())
884 fmt.Printf(" surprisal_variance: %s (%.4f)\n", s.SurprisalVariance.String(), s.SurprisalVariance.Float64())
885 fmt.Printf(" burstiness_gini: %s (%.4f)\n", s.BurstinessGini.String(), s.BurstinessGini.Float64())
886 fmt.Printf(" vertex_coverage: %s (%.4f)\n", s.VertexCoverage.String(), s.VertexCoverage.Float64())
887 fmt.Printf(" avg_walk_distance: %s (%.4f)\n", s.AvgWalkDistance.String(), s.AvgWalkDistance.Float64())
888 fmt.Printf(" bond_rate: %s (%.4f)\n", s.BondRate.String(), s.BondRate.Float64())
889 fmt.Printf(" new_vertex_rate: %s (%.6f)\n", s.NewVertexRate.String(), s.NewVertexRate.Float64())
890 fmt.Printf(" transition_entropy: %s (%.4f)\n", s.TransitionEntropy.String(), s.TransitionEntropy.Float64())
891
892 // Also output machine-readable JSON.
893 if data, err := json.Marshal(s); err == nil {
894 fmt.Printf(" json: %s\n", string(data))
895 }
896 }
897
898