main.go raw
1 // Command socialcuriosity explores the Nostr social graph to feed iskra's
2 // morpheme lattice. Starting from a seed npub, it fetches all kind-1 notes,
3 // digests them through the morpheme enzyme, and follows interaction edges
4 // (reply threads, mentions) to discover new authors.
5 //
6 // When the bond rate for the current npub drops, it looks at who they
7 // corresponded with and picks the most promising unvisited npub. The lattice
8 // grows dynamically when occupancy exceeds 80%.
9 package main
10
11 import (
12 "context"
13 "crypto/rand"
14 "encoding/binary"
15 "encoding/json"
16 "flag"
17 "fmt"
18 "log"
19 "os"
20 "os/signal"
21 "strings"
22 "time"
23
24 "git.mleku.dev/mleku/dendrite/pkg/axiom"
25 "git.mleku.dev/mleku/dendrite/pkg/forage"
26 "git.mleku.dev/mleku/dendrite/pkg/grammar"
27 "git.mleku.dev/mleku/dendrite/pkg/grow"
28 "git.mleku.dev/mleku/dendrite/pkg/lattice"
29 "git.mleku.dev/mleku/dendrite/pkg/mindsicle"
30 "git.mleku.dev/mleku/dendrite/pkg/nostr"
31 "git.mleku.dev/mleku/dendrite/pkg/reflex"
32 "git.mleku.dev/mleku/dendrite/pkg/relay"
33 )
34
35 type socialCheckpoint struct {
36 Round int `json:"round"`
37 CurrentPubkey string `json:"current_pubkey"`
38 SeenNpubs map[string]*forage.NpubScore `json:"seen_npubs"`
39 AllInteractions map[string][]forage.InteractionEdge `json:"all_interactions"`
40 TotalBonded int `json:"total_bonded"`
41 BondRateEWMA float64 `json:"bond_rate_ewma"`
42 ZeroStreak int `json:"zero_streak"`
43 Seed uint64 `json:"seed"`
44 SavedAt string `json:"saved_at"`
45 }
46
47 func main() {
48 var (
49 npubFlag = flag.String("npub", "", "seed npub (required)")
50 relayFlag = flag.String("relay", "wss://relay.orly.dev", "seed relay URL")
51 latticePath = flag.String("lattice", ".condense_morpheme_fast_db.mindsicle.bin", "path to morpheme mindsicle")
52 arcsPath = flag.String("arcs", ".condense_rune_db.arcs.bin", "path to reflex arcs binary")
53 maxEvents = flag.Int("max-events", 500, "max events to fetch per npub")
54 minContent = flag.Int("min-content", 30, "minimum note content length in bytes")
55 zeroLimit = flag.Int("zero-streak", 5, "stop after N consecutive zero-bond npubs")
56 fetchRate = flag.Duration("rate", 3*time.Second, "delay between relay requests")
57 seed = flag.Uint64("seed", 0, "random seed (0=time-based)")
58 outputDB = flag.String("output", ".socialcuriosity_db", "checkpoint file prefix")
59 journalPath = flag.String("journal", "socialcuriosity_journal.log", "audit log path")
60 )
61 flag.Parse()
62
63 if *npubFlag == "" {
64 log.Fatal("socialcuriosity: -npub is required")
65 }
66
67 seedHex, err := nostr.NpubToHex(*npubFlag)
68 if err != nil {
69 log.Fatalf("socialcuriosity: invalid npub: %v", err)
70 }
71 log.Printf("socialcuriosity: seed pubkey %s", seedHex)
72
73 if *seed == 0 {
74 *seed = uint64(time.Now().UnixNano())
75 }
76
77 ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
78 defer cancel()
79
80 // --- Load reflex arcs ---
81 log.Printf("socialcuriosity: loading arcs from %s", *arcsPath)
82 arcFile, err := os.Open(*arcsPath)
83 if err != nil {
84 log.Fatalf("socialcuriosity: cannot open arcs: %v", err)
85 }
86 arcs, err := reflex.Load(arcFile)
87 arcFile.Close()
88 if err != nil {
89 log.Fatalf("socialcuriosity: cannot load arcs: %v", err)
90 }
91 log.Printf("socialcuriosity: loaded %d arcs", len(arcs))
92
93 arcIdx := reflex.BuildArcIndex(arcs)
94 enzyme := reflex.NewMorphemeEnzyme(arcIdx)
95
96 cf := func(tag string) axiom.Constraint {
97 return grammar.NewConstraint(tag, grammar.MorphemeText)
98 }
99
100 // --- Load lattice ---
101 log.Printf("socialcuriosity: loading lattice from %s", *latticePath)
102 f, err := os.Open(*latticePath)
103 if err != nil {
104 log.Fatalf("socialcuriosity: cannot open lattice: %v", err)
105 }
106 lat, _, err := mindsicle.StreamThaw(f, cf)
107 f.Close()
108 if err != nil {
109 log.Fatalf("socialcuriosity: cannot thaw lattice: %v", err)
110 }
111
112 h := lat.Health()
113 log.Printf("socialcuriosity: lattice loaded: %d/%d occupied (%.1f%%)",
114 h.Occupied, h.NodeCount, h.OccupancyRate.Float64()*100)
115
116 lat.BuildAndAttachWalkColumns()
117 wc := lat.GetWalkColumns()
118
119 // --- Load or create checkpoint ---
120 ckptPath := *outputDB + ".checkpoint.json"
121 var ckpt socialCheckpoint
122 if data, err := os.ReadFile(ckptPath); err == nil {
123 if err := json.Unmarshal(data, &ckpt); err == nil {
124 log.Printf("socialcuriosity: resumed from round %d (%d npubs seen, streak=%d)",
125 ckpt.Round, len(ckpt.SeenNpubs), ckpt.ZeroStreak)
126 *seed = ckpt.Seed
127 }
128 }
129 if ckpt.SeenNpubs == nil {
130 ckpt.SeenNpubs = make(map[string]*forage.NpubScore)
131 }
132 if ckpt.AllInteractions == nil {
133 ckpt.AllInteractions = make(map[string][]forage.InteractionEdge)
134 }
135 ckpt.Seed = *seed
136
137 if ckpt.CurrentPubkey == "" {
138 ckpt.CurrentPubkey = seedHex
139 }
140
141 // --- Open journal ---
142 journal, err := os.OpenFile(*journalPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
143 if err != nil {
144 log.Fatalf("socialcuriosity: cannot open journal: %v", err)
145 }
146 defer journal.Close()
147 fmt.Fprintf(journal, "# socialcuriosity session started at %s (seed=%d, npub=%s)\n",
148 time.Now().Format(time.RFC3339), *seed, *npubFlag)
149
150 // --- Create relay pool ---
151 pool := relay.NewPool(*relayFlag)
152 log.Printf("socialcuriosity: relay pool seeded with %s", *relayFlag)
153
154 // --- Main loop ---
155 startRound := ckpt.Round + 1
156 log.Printf("socialcuriosity: starting from round %d (current=%s, zero-limit=%d)",
157 startRound, ckpt.CurrentPubkey[:16], *zeroLimit)
158
159 for round := startRound; ; round++ {
160 select {
161 case <-ctx.Done():
162 log.Printf("socialcuriosity: interrupted at round %d, saving", round)
163 saveCheckpoint(ckptPath, ckpt)
164 saveLattice(lat, *latticePath)
165 return
166 default:
167 }
168
169 currentHex := ckpt.CurrentPubkey
170 npubStr, _ := nostr.PubkeyToNpub(currentHex)
171 log.Printf("socialcuriosity: round %d: feeding from %s (%s)",
172 round, npubStr, currentHex[:16])
173
174 // 1. Fetch events for current npub.
175 events, sourceRelay := pool.FetchEventsMultiRelay(
176 ctx, currentHex, *maxEvents, *minContent, 15*time.Second)
177
178 if len(events) == 0 {
179 log.Printf("socialcuriosity: round %d: no events found for %s", round, currentHex[:16])
180 ckpt.ZeroStreak++
181 } else {
182 log.Printf("socialcuriosity: round %d: fetched %d events from %s",
183 round, len(events), sourceRelay)
184
185 // Deduplicate by event ID.
186 events = dedup(events)
187
188 // 2. Digest each note.
189 score, ok := ckpt.SeenNpubs[currentHex]
190 if !ok {
191 score = &forage.NpubScore{Pubkey: currentHex}
192 ckpt.SeenNpubs[currentHex] = score
193 }
194
195 roundBonded := 0
196 roundElems := 0
197
198 for _, ev := range events {
199 elems := enzyme.DigestText(ev.Content)
200 bonded := feedElements(ctx, lat, elems)
201
202 score.UpdateBondRate(len(elems), bonded, len(ev.Content))
203 roundBonded += bonded
204 roundElems += len(elems)
205 }
206
207 ckpt.TotalBonded += roundBonded
208
209 // Session EWMA.
210 sessionRate := 0.0
211 if roundElems > 0 {
212 sessionRate = float64(roundBonded) / float64(roundElems)
213 }
214 ckpt.BondRateEWMA = 0.8*ckpt.BondRateEWMA + 0.2*sessionRate
215
216 if roundBonded == 0 {
217 ckpt.ZeroStreak++
218 } else {
219 ckpt.ZeroStreak = 0
220 }
221
222 log.Printf("socialcuriosity: round %d: %d notes, %d morphemes, %d bonded (%.2f%%), streak=%d",
223 round, len(events), roundElems, roundBonded, sessionRate*100, ckpt.ZeroStreak)
224
225 // 3. Extract interactions.
226 interactions := forage.ExtractInteractions(events, currentHex)
227 ckpt.AllInteractions[currentHex] = interactions
228 if len(interactions) > 0 {
229 log.Printf("socialcuriosity: round %d: %d interaction edges from %s",
230 round, len(interactions), currentHex[:16])
231 }
232
233 // 4. Journal.
234 fmt.Fprintf(journal, "%s round=%d pubkey=%s npub=%s relay=%s events=%d elems=%d bonded=%d rate=%.2f%% bonds_per_kb=%.1f streak=%d interactions=%d\n",
235 time.Now().Format(time.RFC3339), round, currentHex, npubStr, sourceRelay,
236 len(events), roundElems, roundBonded, sessionRate*100,
237 score.BondsPerKB, ckpt.ZeroStreak, len(interactions))
238 }
239
240 // 5. Dissolve weak bonds.
241 dissolved := dissolveWeak(lat)
242 if dissolved > 0 {
243 log.Printf("socialcuriosity: dissolved %d weak bonds", dissolved)
244 }
245
246 // 6. Update health.
247 h = lat.Health()
248 log.Printf("socialcuriosity: health: %dK/%dK (%.1f%%)",
249 h.Occupied/1000, h.NodeCount/1000, h.OccupancyRate.Float64()*100)
250
251 // 7. Dynamic growth at 80% occupancy.
252 if h.OccupancyRate.Float64() > 0.80 {
253 growBy := h.NodeCount / 2
254 var expandSeed [32]byte
255 binary.LittleEndian.PutUint64(expandSeed[:], *seed+uint64(round)*0xdead)
256 rand.Read(expandSeed[8:])
257 added := grammar.ExpandLattice(lat, grammar.MorphemeText, growBy, expandSeed, cf, grammar.MorphemeDefaultCounts)
258 if added > 0 {
259 lat.BuildAndAttachWalkColumns()
260 wc = lat.GetWalkColumns()
261 log.Printf("socialcuriosity: expanded lattice by %d nodes (now %d)", added, lat.Size())
262 ckpt.ZeroStreak = 0
263 }
264 }
265
266 // 8. Decide whether to switch npubs.
267 // Walk lattice weak regions for baby talk (used as tiebreaker).
268 walkSeed := *seed + uint64(round)*0xbeef
269 walkedValues := forage.WeakRegionWalk(lat, wc, 5000, walkSeed)
270 babyTalk := forage.ExtractBabyTalk(walkedValues)
271
272 // Merge all interactions for scoring.
273 var allEdges []forage.InteractionEdge
274 for _, edges := range ckpt.AllInteractions {
275 allEdges = append(allEdges, edges...)
276 }
277
278 nextHex := forage.ScoreNextNpub(allEdges, ckpt.SeenNpubs, babyTalk)
279 if nextHex != "" && nextHex != currentHex {
280 nextNpub, _ := nostr.PubkeyToNpub(nextHex)
281 log.Printf("socialcuriosity: switching to %s (%s)", nextNpub, nextHex[:16])
282 ckpt.CurrentPubkey = nextHex
283 } else if len(events) == 0 || (ckpt.SeenNpubs[currentHex] != nil && ckpt.SeenNpubs[currentHex].BondRateEWMA < 0.0001) {
284 // Current npub exhausted and no candidates.
285 if nextHex == "" {
286 log.Printf("socialcuriosity: no unvisited npubs remaining")
287 ckpt.ZeroStreak = *zeroLimit // force convergence
288 }
289 }
290
291 // 9. Checkpoint every 5 rounds.
292 ckpt.Round = round
293 if round%5 == 0 {
294 saveCheckpoint(ckptPath, ckpt)
295 saveLattice(lat, *latticePath)
296 log.Printf("socialcuriosity: checkpoint saved at round %d", round)
297 }
298
299 // 10. Check convergence.
300 if ckpt.ZeroStreak >= *zeroLimit {
301 log.Printf("socialcuriosity: converged — %d consecutive zero-bond rounds", ckpt.ZeroStreak)
302 break
303 }
304
305 // Rate limit.
306 select {
307 case <-time.After(*fetchRate):
308 case <-ctx.Done():
309 }
310 }
311
312 // Final save.
313 saveCheckpoint(ckptPath, ckpt)
314 saveLattice(lat, *latticePath)
315
316 h = lat.Health()
317 fmt.Fprintf(journal, "# socialcuriosity session ended at %s: %d rounds, %d npubs visited, %d total bonded, %dK/%dK occupied (%.1f%%)\n",
318 time.Now().Format(time.RFC3339), ckpt.Round, len(ckpt.SeenNpubs),
319 ckpt.TotalBonded, h.Occupied/1000, h.NodeCount/1000,
320 h.OccupancyRate.Float64()*100)
321
322 log.Printf("socialcuriosity: done — %d rounds, %d npubs, %d total bonded",
323 ckpt.Round, len(ckpt.SeenNpubs), ckpt.TotalBonded)
324 log.Printf("%s", pool.Stats())
325 }
326
327 // feedElements feeds morpheme elements into the lattice via grow.Run.
328 func feedElements(ctx context.Context, lat *lattice.Lattice, elems []axiom.Element) int {
329 if len(elems) == 0 {
330 return 0
331 }
332
333 ch := make(chan axiom.Element, len(elems))
334 go func() {
335 for _, e := range elems {
336 ch <- e
337 }
338 close(ch)
339 }()
340
341 cfg := grow.Config{
342 MaxSteps: 10,
343 Workers: grow.WorkerCount(),
344 }
345
346 events := make(chan grow.Event, 256)
347 bonded := 0
348 done := make(chan struct{})
349 go func() {
350 for ev := range events {
351 if ev.Type == grow.EventBonded {
352 bonded++
353 }
354 }
355 close(done)
356 }()
357
358 grow.Run(ctx, lat, ch, cfg, events)
359 close(events)
360 <-done
361 return bonded
362 }
363
364 // dissolveWeak removes occupants with low contextual lock-in.
365 func dissolveWeak(l *lattice.Lattice) int {
366 nodes := l.Nodes()
367 target := max(1, len(nodes)/100)
368
369 type scored struct {
370 node *lattice.Node
371 score float64
372 }
373 var weak []scored
374 for _, n := range nodes {
375 if !n.Occupied() {
376 continue
377 }
378 s := n.ContextualLockIn().Float64()
379 if s < 0.3 {
380 weak = append(weak, scored{n, s})
381 }
382 }
383
384 dissolved := 0
385 for _, s := range weak {
386 if dissolved >= target {
387 break
388 }
389 s.node.Dissolve()
390 l.ReindexVacant(s.node)
391 dissolved++
392 }
393 return dissolved
394 }
395
396 // dedup removes duplicate events by ID.
397 func dedup(events []*nostr.Event) []*nostr.Event {
398 seen := make(map[string]bool, len(events))
399 out := make([]*nostr.Event, 0, len(events))
400 for _, ev := range events {
401 if !seen[ev.ID] {
402 seen[ev.ID] = true
403 out = append(out, ev)
404 }
405 }
406 return out
407 }
408
409 func saveCheckpoint(path string, ckpt socialCheckpoint) {
410 ckpt.SavedAt = time.Now().Format(time.RFC3339)
411 data, err := json.MarshalIndent(ckpt, "", " ")
412 if err != nil {
413 log.Printf("socialcuriosity: WARNING: cannot marshal checkpoint: %v", err)
414 return
415 }
416 tmp := path + ".tmp"
417 if err := os.WriteFile(tmp, data, 0o644); err != nil {
418 log.Printf("socialcuriosity: WARNING: cannot write checkpoint: %v", err)
419 return
420 }
421 os.Rename(tmp, path)
422 }
423
424 func saveLattice(lat *lattice.Lattice, path string) {
425 tmp := path + ".tmp"
426 f, err := os.Create(tmp)
427 if err != nil {
428 log.Printf("socialcuriosity: WARNING: cannot create %s: %v", tmp, err)
429 return
430 }
431 if err := mindsicle.StreamFreeze(lat, nil, f); err != nil {
432 f.Close()
433 os.Remove(tmp)
434 log.Printf("socialcuriosity: WARNING: save failed: %v", err)
435 return
436 }
437 f.Close()
438 if err := os.Rename(tmp, path); err != nil {
439 log.Printf("socialcuriosity: WARNING: rename failed: %v", err)
440 return
441 }
442 fi, _ := os.Stat(path)
443 if fi != nil {
444 log.Printf("socialcuriosity: saved %.2f MB", float64(fi.Size())/(1024*1024))
445 }
446 }
447
448 func truncate(s string, n int) string {
449 s = strings.ReplaceAll(s, "\n", " ")
450 if len(s) > n {
451 return s[:n] + "..."
452 }
453 return s
454 }
455