package forage import ( "context" "crypto/sha256" "encoding/hex" "fmt" "io" "net/http" "os" "strings" "sync" "time" ) // Forager handles internet content acquisition with safety constraints. type Forager struct { // DomainWhitelist restricts which domains the organism can fetch from. // Empty means deny all. DomainWhitelist map[string]bool // MaxBytesPerGen is the bandwidth cap per generation. MaxBytesPerGen int64 // LogPath is where fetched resource hashes are logged. LogPath string mu sync.Mutex bytesUsed int64 fetchCount int gen int } // FetchResult captures a single fetch operation. type FetchResult struct { URL string ContentType string Body []byte Hash string // SHA-256 of the body Duration time.Duration StatusCode int } // NewForager creates a forager with safe defaults. func NewForager() *Forager { return &Forager{ DomainWhitelist: map[string]bool{ "go.dev": true, "pkg.go.dev": true, "github.com": true, "raw.githubusercontent.com": true, "nostr.com": true, "nips.nostr.com": true, }, MaxBytesPerGen: 1 << 20, // 1MB LogPath: "_output/forage_log.md", } } // SetGeneration resets per-generation counters. func (f *Forager) SetGeneration(gen int) { f.mu.Lock() defer f.mu.Unlock() f.gen = gen f.bytesUsed = 0 f.fetchCount = 0 } // URLForNeed constructs a URL from a Need. Known URL patterns are used // first; unknown needs return empty (should go to oracle instead). func URLForNeed(need Need) string { topic := need.Topic // NIP specifications have known URLs. if strings.HasPrefix(topic, "NIP-") || strings.HasPrefix(topic, "nip-") { num := strings.TrimPrefix(strings.TrimPrefix(topic, "NIP-"), "nip-") return fmt.Sprintf("https://raw.githubusercontent.com/nostr-protocol/nips/master/%s.md", num) } // Go standard library docs. if strings.Contains(topic, "/") && !strings.Contains(topic, " ") { return fmt.Sprintf("https://pkg.go.dev/%s", topic) } return "" } // Fetch retrieves content from a URL with safety checks. func (f *Forager) Fetch(ctx context.Context, url string) (*FetchResult, error) { // Domain whitelist check. domain := extractDomain(url) if !f.DomainWhitelist[domain] { return nil, fmt.Errorf("domain %q not in whitelist", domain) } // Bandwidth check. f.mu.Lock() if f.bytesUsed >= f.MaxBytesPerGen { f.mu.Unlock() return nil, fmt.Errorf("bandwidth cap reached: %d bytes used in gen %d", f.bytesUsed, f.gen) } f.mu.Unlock() start := time.Now() req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { return nil, err } req.Header.Set("User-Agent", "dendrite/0.1 (lattice organism)") resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } defer resp.Body.Close() // Read with bandwidth limit. f.mu.Lock() remaining := f.MaxBytesPerGen - f.bytesUsed f.mu.Unlock() body, err := io.ReadAll(io.LimitReader(resp.Body, remaining)) if err != nil { return nil, err } // Update bandwidth counter. f.mu.Lock() f.bytesUsed += int64(len(body)) f.fetchCount++ f.mu.Unlock() // Hash for audit trail. h := sha256.Sum256(body) hash := hex.EncodeToString(h[:]) result := &FetchResult{ URL: url, ContentType: resp.Header.Get("Content-Type"), Body: body, Hash: hash, Duration: time.Since(start), StatusCode: resp.StatusCode, } // Log the fetch. f.logFetch(result) return result, nil } // extractDomain extracts the hostname from a URL. func extractDomain(url string) string { // Strip scheme. u := url if i := strings.Index(u, "://"); i >= 0 { u = u[i+3:] } // Strip path. if i := strings.Index(u, "/"); i >= 0 { u = u[:i] } // Strip port. if i := strings.LastIndex(u, ":"); i >= 0 { u = u[:i] } return u } // logFetch appends a fetch record to the forage log. func (f *Forager) logFetch(result *FetchResult) { file, err := os.OpenFile(f.LogPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) if err != nil { return } defer file.Close() fmt.Fprintf(file, "- gen=%d url=%s status=%d size=%d hash=%s duration=%s\n", f.gen, result.URL, result.StatusCode, len(result.Body), result.Hash[:16], result.Duration.Round(time.Millisecond)) } // BytesRemaining returns how many bytes can still be fetched this generation. func (f *Forager) BytesRemaining() int64 { f.mu.Lock() defer f.mu.Unlock() r := f.MaxBytesPerGen - f.bytesUsed if r < 0 { return 0 } return r }