package main // Profile Worker - owns all author profile data and subscription management. // // Data authority: profSt.authorNames/Pics/Content/Ts/Relays/Follows/Mutes (all profiles). // Shell keeps: profSt.authorNames/Pics view cache (small, for sync render), followSet/muteSet // (sync filter derived sets), myFollows/myMutes (own lists for action operations). // // Wire: supervisor <-> worker // supervisor -> worker: // ["P_RESOLVE", pk] -- request fetch if not already fetched // ["P_RESOLVE_FORCE", pk] -- re-fetch unconditionally // ["P_EVENT", evJSON] -- kind 0/3/10002/10000 event from ap-* or prof sub // ["P_EOSE", subID] -- ap-* EOSE (retry logic) // ["P_SET_RELAYS", urlsJSON] // ["P_SET_PUBKEY", pk] // ["P_IDB_LOADED", pksJSON] -- Shell finished IDB load; mark these pks fetched // // worker -> supervisor (Shell or relay-proxy): // ["P_RESOLVED", pk, name, pic] -- kind 0 resolved // ["P_CONTENT", pk, contentJSON] -- full kind 0 content // ["P_FOLLOWS", pk, pksJSON] -- kind 3 follow list // ["P_MUTES", pk, pksJSON] -- kind 10000 mute list // ["P_RELAYS", pk, urlsJSON] -- kind 10002 relay list // ["P_SUB", subID, filterJSON, urlsJSON] // ["P_CLOSE", subID] // ["P_RETRY_REQ"] -- request Shell's list of still-missing pks import ( "runtime" "git.smesh.lol/musiquay/web/common/accum" "git.smesh.lol/musiquay/web/common/helpers" "git.smesh.lol/musiquay/web/common/jsbridge/profile" "git.smesh.lol/musiquay/web/common/mw" "git.smesh.lol/nostr/pkg/core" ) const rotateAuthorMax = 500 // retryMax is the per-pk attempt cap. Each attempt waits sweepIntervalMs. const retryMax = 6 const sweepIntervalMs = 15000 var discoveryRelays []string // profileState is the profile // Package globals are immutable outside initialization, so it lives in // one self-mutating type reached through a package-level pointer. type profileState struct { // Operational state (fetch queue management) fetchedK0 map[string]bool fetchedK10k map[string]bool resolvedSet map[string]bool // pk -> kind 0 resolved with a name requestedSet map[string]int32 // pk -> retry attempts so far (capped at retryMax) authorSubPK map[string]string fetchQueue accum.Strings fetchTimer int32 retryRound int32 retryTimer int32 sweepTimer int32 subCounter int32 relayURLs accum.Strings myPK string // Relay discovery (two-gen rotation with author caches) authorRelays map[string][]string authorRelaysOld map[string][]string relayFreq map[string]int32 relayFreqOld map[string]int32 // Profile data (canonical authority - two-generation rotation) authorNames map[string]string authorNamesOld map[string]string authorPics map[string]string authorPicsOld map[string]string authorContent map[string]string authorContentOld map[string]string authorTs map[string]int64 authorTsOld map[string]int64 authorFollows map[string][]string authorMutes map[string][]string authorCacheCount int32 } var profSt *profileState func initState() { if profSt != nil { return } // profSt and everything it holds live as long as this worker: build them // in the root arena, not in this call's frame arena which dies on // return. runtime.SovereignSetArena(runtime.RootArena()) profSt = &profileState{} profSt.fetchedK0 = map[string]bool{} profSt.fetchedK10k = map[string]bool{} profSt.resolvedSet = map[string]bool{} profSt.requestedSet = map[string]int32{} profSt.authorSubPK = map[string]string{} profSt.authorRelays = map[string][]string{} profSt.authorRelaysOld = map[string][]string{} profSt.relayFreq = map[string]int32{} profSt.relayFreqOld = map[string]int32{} profSt.authorNames = map[string]string{} profSt.authorNamesOld = map[string]string{} profSt.authorPics = map[string]string{} profSt.authorPicsOld = map[string]string{} profSt.authorContent = map[string]string{} profSt.authorContentOld = map[string]string{} profSt.authorTs = map[string]int64{} profSt.authorTsOld = map[string]int64{} profSt.authorFollows = map[string][]string{} profSt.authorMutes = map[string][]string{} runtime.SovereignRestoreArena(runtime.RootArena()) } func initProfileGlobals() { discoveryRelays = []string{ "wss://purplepag.es", "wss://relay.damus.io", "wss://nos.lol", } } func main() { initProfileGlobals() initState() profile.WorkerOnMessage(handleMessage) scheduleSweep() } // scheduleSweep arms the periodic retry timer. Every sweepIntervalMs the // worker re-fetches any pks in profSt.requestedSet that aren't in profSt.resolvedSet, until // each pk hits retryMax. This is independent of EOSE-based retry and works // even when subs were closed without producing the expected event. func scheduleSweep() { if profSt.sweepTimer != 0 { profile.WorkerClearTimeout(profSt.sweepTimer) } profSt.sweepTimer = profile.WorkerSetTimeout(int32(sweepIntervalMs), func() { profSt.sweepTimer = 0 runSweep() scheduleSweep() }) } func runSweep() { var missing []string for pk, attempts := range profSt.requestedSet { if profSt.resolvedSet[pk] { continue } if attempts >= retryMax { continue } missing = push(missing, pk) } if len(missing) == 0 { return } for _, pk := range missing { profSt.requestedSet[pk] = profSt.requestedSet[pk] + 1 profSt.fetchedK0[pk] = false } proxy := buildProxyURLs(topRelays(8)) const batchSize = 100 for i := 0; i < len(missing); i += batchSize { end := i + batchSize if end > len(missing) { end = len(missing) } chunk := missing[i:end] authors := buildAuthorsJSON(chunk) limit := helpers.Itoa(int64(len(chunk) * 2)) profSt.subCounter++ emitSub("ap-sweep-" | helpers.Itoa(int64(profSt.subCounter)), `{"authors":` | authors | `,"kinds":[0,10002],"limit":` | limit | `}`, proxy) profSt.subCounter++ emitSub("ap-d-" | helpers.Itoa(int64(profSt.subCounter)), `{"authors":` | authors | `,"kinds":[0],"limit":` | helpers.Itoa(int64(len(chunk))) | `}`, profSt.relayURLs.Slice()) } } func rotateAuthorCaches() { profSt.authorNamesOld = profSt.authorNames profSt.authorNames = map[string]string{} profSt.authorPicsOld = profSt.authorPics profSt.authorPics = map[string]string{} profSt.authorContentOld = profSt.authorContent profSt.authorContent = map[string]string{} profSt.authorTsOld = profSt.authorTs profSt.authorTs = map[string]int64{} profSt.authorRelaysOld = profSt.authorRelays profSt.authorRelays = map[string][]string{} profSt.relayFreqOld = profSt.relayFreq profSt.relayFreq = map[string]int32{} // Clear fetch tracking maps on rotation. If we kept these populated while // the data caches were thrown out, the worker would refuse to re-fetch // authors whose data was rotated away ("we already fetched X, but the // cache is empty for X" → permanent blank). profSt.fetchedK0 = map[string]bool{} profSt.fetchedK10k = map[string]bool{} profSt.resolvedSet = map[string]bool{} profSt.requestedSet = map[string]int32{} profSt.authorCacheCount = 0 } func handleMessage(msg string) { w := mw.New(msg) switch w.Str() { case "P_RESOLVE": pk := w.Str() queueProfileFetch(pk) case "P_RESOLVE_FORCE": pk := w.Str() profSt.fetchedK0[pk] = false queueProfileFetch(pk) case "P_EVENT": evJSON := w.Raw() ev := nostr.ParseEvent(evJSON) if ev != nil { applyEvent(ev) } case "P_EOSE": subID := w.Str() handleEOSE(subID) case "P_SET_RELAYS": profSt.relayURLs.Set(parseStringArray(w.Raw())) case "P_SET_PUBKEY": profSt.myPK = w.Str() case "P_IDB_LOADED": // Shell finished loading profiles from musiquay-kv IDB. Mark them fetched AND // resolved so Profile Worker doesn't waste relay bandwidth re-fetching them. pksJSON := w.Raw() pks := parseStringArray(pksJSON) for _, pk := range pks { profSt.fetchedK0[pk] = true profSt.resolvedSet[pk] = true } case "P_RETRY_PROVIDE": // Shell sent list of pending pubkeys without resolved names. Re-queue // them, the periodic sweep + queue flush will pick them up. We bypass // per-pk attempt cap here because this is a user-driven retry signal // (tab switch, refresh button) not an automatic background retry. pksJSON := w.Raw() pks := parseStringArray(pksJSON) for _, pk := range pks { if profSt.resolvedSet[pk] { continue } profSt.fetchedK0[pk] = false if _, ok2 := profSt.requestedSet[pk]; ok2 { profSt.requestedSet[pk] = 0 // reset attempts on user-driven retry } else { profSt.requestedSet[pk] = 0 } queueProfileFetch(pk) } } } func applyEvent(ev *nostr.Event) { switch ev.Kind { case 0: applyKind0(ev) case 3: applyKind3(ev) case 10000: applyKind10000(ev) case 10002: applyKind10002(ev) } } func applyKind0(ev *nostr.Event) { prevTs, hasPrev := profSt.authorTs[ev.PubKey] if hasPrev && ev.CreatedAt <= prevTs { return } if _, exists := profSt.authorContent[ev.PubKey]; !exists { profSt.authorCacheCount++ if profSt.authorCacheCount > rotateAuthorMax { rotateAuthorCaches() } } profSt.authorTs[ev.PubKey] = ev.CreatedAt profSt.authorContent[ev.PubKey] = ev.Content name := helpers.JsonGetString(ev.Content, "name") if len(name) == 0 { name = helpers.JsonGetString(ev.Content, "display_name") } pic := helpers.JsonGetString(ev.Content, "picture") if len(name) > 0 { profSt.authorNames[ev.PubKey] = name profSt.resolvedSet[ev.PubKey] = true } if len(pic) > 0 { profSt.authorPics[ev.PubKey] = pic } // Emit P_RESOLVED so Shell can fill note header placeholders. profile.WorkerPost(`["P_RESOLVED",` | jstr(ev.PubKey) | `,` | jstr(name) | `,` | jstr(pic) | `]`) // Emit P_CONTENT so Shell can update profile page and zap buttons. if len(ev.Content) > 0 { profile.WorkerPost(`["P_CONTENT",` | jstr(ev.PubKey) | `,` | helpers.JsonString(ev.Content) | `]`) } } func applyKind3(ev *nostr.Event) { var pks []string for _, tag := range nostr.TagsGetAll(ev.Tags, "p") { if v := nostr.TagValue(tag); v != "" { pks = push(pks, v) } } profSt.authorFollows[ev.PubKey] = pks profile.WorkerPost(`["P_FOLLOWS",` | jstr(ev.PubKey) | `,` | buildAuthorsJSON(pks) | `]`) } func applyKind10000(ev *nostr.Event) { var pks []string for _, tag := range nostr.TagsGetAll(ev.Tags, "p") { if v := nostr.TagValue(tag); v != "" { pks = push(pks, v) } } profSt.authorMutes[ev.PubKey] = pks profile.WorkerPost(`["P_MUTES",` | jstr(ev.PubKey) | `,` | buildAuthorsJSON(pks) | `]`) } func applyKind10002(ev *nostr.Event) { tags := nostr.TagsGetAll(ev.Tags, "r") if tags == nil { return } var urls []string for _, tag := range tags { u := string(nostr.TagValue(tag)) if u != "" { urls = push(urls, u) profSt.relayFreq[u] = profSt.relayFreq[u] + 1 } } if len(urls) > 0 { profSt.authorRelays[ev.PubKey] = urls profile.WorkerPost(`["P_RELAYS",` | jstr(ev.PubKey) | `,` | buildURLsJSON(urls) | `]`) } } func queueProfileFetch(pk string) { if len(pk) != 64 { return } // Track every requested pk so the periodic sweep can retry until // resolved or per-pk attempt cap is hit. if _, ok := profSt.requestedSet[pk]; !ok { profSt.requestedSet[pk] = 0 } if profSt.fetchedK0[pk] { return } profSt.fetchedK0[pk] = true profSt.fetchQueue.Push(pk) if profSt.fetchTimer != 0 { profile.WorkerClearTimeout(profSt.fetchTimer) } profSt.fetchTimer = profile.WorkerSetTimeout(300, func() { profSt.fetchTimer = 0 flushFetchQueue() }) } func flushFetchQueue() { if profSt.fetchQueue.Len() == 0 { return } queue := profSt.fetchQueue.Slice() profSt.fetchQueue.Reset() proxy := buildProxyURLs(nil) const batchSize = 100 for i := 0; i < len(queue); i += batchSize { end := i + batchSize if end > len(queue) { end = len(queue) } chunk := queue[i:end] authors := buildAuthorsJSON(chunk) limit := helpers.Itoa(int64(len(chunk))) profSt.subCounter++ emitSub("ap-batch-q-" | helpers.Itoa(int64(profSt.subCounter)), `{"authors":` | authors | `,"kinds":[0],"limit":` | limit | `}`, proxy) profSt.subCounter++ emitSub("ap-d-" | helpers.Itoa(int64(profSt.subCounter)), `{"authors":` | authors | `,"kinds":[0],"limit":` | limit | `}`, profSt.relayURLs.Slice()) } } func fetchAuthorProfile(pk string) { if profSt.fetchedK0[pk] { return } profSt.fetchedK0[pk] = true profSt.subCounter++ subID := "ap-" | helpers.Itoa(int64(profSt.subCounter)) profSt.authorSubPK[subID] = pk emitSub(subID, `{"authors":[` | jstr(pk) | `],"kinds":[0,3,10002,10000],"limit":6}`, buildProxy(pk)) } func handleEOSE(subID string) { // "ap-batch-" prefix matches both "ap-batch-q-N" (queue flush) and // "ap-batch-N" (legacy P_RETRY_PROVIDE re-batch). The periodic sweep // now handles per-pk retries with its own cap, so EOSE-triggered retry // is no longer needed. Keeping it here would double-fetch and the // global profSt.retryRound cap caused unresolved pks to permanently stick. if len(subID) > 9 && subID[:9] == "ap-batch-" { return } if len(subID) > 9 && subID[:9] == "ap-sweep-" { return } if len(subID) > 3 && subID[:3] == "ap-" { pk, ok := profSt.authorSubPK[subID] if !ok { return } delete(profSt.authorSubPK, subID) if !profSt.resolvedSet[pk] { rels, ok2 := profSt.authorRelays[pk] if !ok2 { rels, ok2 = profSt.authorRelaysOld[pk] } if ok2 && len(rels) > 0 && !profSt.fetchedK10k[pk] { profSt.fetchedK10k[pk] = true profSt.fetchedK0[pk] = false fetchAuthorProfile(pk) } } } } func emitSub(subID, filterJSON string, urls []string) { profile.WorkerPost(`["P_SUB",` | jstr(subID) | `,` | filterJSON | `,` | buildURLsJSON(urls) | `]`) } func buildProxy(pk string) (ss []string) { out := []string{:len(discoveryRelays)} copy(out, discoveryRelays) for i := int32(0); i < profSt.relayURLs.Len(); i++ { out = appendUnique(out, profSt.relayURLs.At(i)) } if rels, ok2 := profSt.authorRelays[pk]; ok2 { for _, r := range rels { out = appendUnique(out, r) } } else if oldRels, oldOK := profSt.authorRelaysOld[pk]; oldOK { for _, r := range oldRels { out = appendUnique(out, r) } } for _, r := range topRelays(4) { out = appendUnique(out, r) } return out } func buildProxyURLs(extra []string) (ss []string) { out := []string{:len(discoveryRelays)} copy(out, discoveryRelays) for i := int32(0); i < profSt.relayURLs.Len(); i++ { out = appendUnique(out, profSt.relayURLs.At(i)) } for _, u := range extra { out = appendUnique(out, u) } return out } func topRelays(n int32) (ss []string) { type kv struct{ url string; count int32 } var all []kv for url, count := range profSt.relayFreq { all = push(all, kv{url, count}) } for i := 0; i < len(all); i++ { for j := i + 1; j < len(all); j++ { if all[j].count > all[i].count { all[i], all[j] = all[j], all[i] } } } var result []string for i := 0; i < len(all) && i < n; i++ { result = push(result, all[i].url) } return result } func appendUnique(list []string, val string) (ss []string) { for _, v := range list { if v == val { return list } } return push(list, val) } func buildAuthorsJSON(pks []string) (s string) { s := "[" for i, pk := range pks { if i > 0 { s = s | "," } s = s | jstr(pk) } return s | "]" } func buildURLsJSON(urls []string) (s string) { s := "[" for i, u := range urls { if i > 0 { s = s | "," } s = s | jstr(u) } return s | "]" } func parseStringArray(json string) (ss []string) { var out []string i := 0 for i < len(json) && json[i] != '[' { i++ } if i >= len(json) { return nil } i++ for { for i < len(json) && (json[i] == ' ' || json[i] == ',' || json[i] == '\n') { i++ } if i >= len(json) || json[i] == ']' { break } if json[i] != '"' { break } i++ start := i for i < len(json) && json[i] != '"' { if json[i] == '\\' { i++ } i++ } if i >= len(json) { break } out = push(out, json[start:i]) i++ } return out } func jstr(s string) (sv string) { return helpers.JsonString(s) }