fetch.go raw
1 package forage
2
3 import (
4 "context"
5 "crypto/sha256"
6 "encoding/hex"
7 "fmt"
8 "io"
9 "net/http"
10 "os"
11 "strings"
12 "sync"
13 "time"
14 )
15
16 // Forager handles internet content acquisition with safety constraints.
17 type Forager struct {
18 // DomainWhitelist restricts which domains the organism can fetch from.
19 // Empty means deny all.
20 DomainWhitelist map[string]bool
21
22 // MaxBytesPerGen is the bandwidth cap per generation.
23 MaxBytesPerGen int64
24
25 // LogPath is where fetched resource hashes are logged.
26 LogPath string
27
28 mu sync.Mutex
29 bytesUsed int64
30 fetchCount int
31 gen int
32 }
33
34 // FetchResult captures a single fetch operation.
35 type FetchResult struct {
36 URL string
37 ContentType string
38 Body []byte
39 Hash string // SHA-256 of the body
40 Duration time.Duration
41 StatusCode int
42 }
43
44 // NewForager creates a forager with safe defaults.
45 func NewForager() *Forager {
46 return &Forager{
47 DomainWhitelist: map[string]bool{
48 "go.dev": true,
49 "pkg.go.dev": true,
50 "github.com": true,
51 "raw.githubusercontent.com": true,
52 "nostr.com": true,
53 "nips.nostr.com": true,
54 },
55 MaxBytesPerGen: 1 << 20, // 1MB
56 LogPath: "_output/forage_log.md",
57 }
58 }
59
60 // SetGeneration resets per-generation counters.
61 func (f *Forager) SetGeneration(gen int) {
62 f.mu.Lock()
63 defer f.mu.Unlock()
64 f.gen = gen
65 f.bytesUsed = 0
66 f.fetchCount = 0
67 }
68
69 // URLForNeed constructs a URL from a Need. Known URL patterns are used
70 // first; unknown needs return empty (should go to oracle instead).
71 func URLForNeed(need Need) string {
72 topic := need.Topic
73
74 // NIP specifications have known URLs.
75 if strings.HasPrefix(topic, "NIP-") || strings.HasPrefix(topic, "nip-") {
76 num := strings.TrimPrefix(strings.TrimPrefix(topic, "NIP-"), "nip-")
77 return fmt.Sprintf("https://raw.githubusercontent.com/nostr-protocol/nips/master/%s.md", num)
78 }
79
80 // Go standard library docs.
81 if strings.Contains(topic, "/") && !strings.Contains(topic, " ") {
82 return fmt.Sprintf("https://pkg.go.dev/%s", topic)
83 }
84
85 return ""
86 }
87
88 // Fetch retrieves content from a URL with safety checks.
89 func (f *Forager) Fetch(ctx context.Context, url string) (*FetchResult, error) {
90 // Domain whitelist check.
91 domain := extractDomain(url)
92 if !f.DomainWhitelist[domain] {
93 return nil, fmt.Errorf("domain %q not in whitelist", domain)
94 }
95
96 // Bandwidth check.
97 f.mu.Lock()
98 if f.bytesUsed >= f.MaxBytesPerGen {
99 f.mu.Unlock()
100 return nil, fmt.Errorf("bandwidth cap reached: %d bytes used in gen %d",
101 f.bytesUsed, f.gen)
102 }
103 f.mu.Unlock()
104
105 start := time.Now()
106
107 req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
108 if err != nil {
109 return nil, err
110 }
111 req.Header.Set("User-Agent", "dendrite/0.1 (lattice organism)")
112
113 resp, err := http.DefaultClient.Do(req)
114 if err != nil {
115 return nil, err
116 }
117 defer resp.Body.Close()
118
119 // Read with bandwidth limit.
120 f.mu.Lock()
121 remaining := f.MaxBytesPerGen - f.bytesUsed
122 f.mu.Unlock()
123
124 body, err := io.ReadAll(io.LimitReader(resp.Body, remaining))
125 if err != nil {
126 return nil, err
127 }
128
129 // Update bandwidth counter.
130 f.mu.Lock()
131 f.bytesUsed += int64(len(body))
132 f.fetchCount++
133 f.mu.Unlock()
134
135 // Hash for audit trail.
136 h := sha256.Sum256(body)
137 hash := hex.EncodeToString(h[:])
138
139 result := &FetchResult{
140 URL: url,
141 ContentType: resp.Header.Get("Content-Type"),
142 Body: body,
143 Hash: hash,
144 Duration: time.Since(start),
145 StatusCode: resp.StatusCode,
146 }
147
148 // Log the fetch.
149 f.logFetch(result)
150
151 return result, nil
152 }
153
154 // extractDomain extracts the hostname from a URL.
155 func extractDomain(url string) string {
156 // Strip scheme.
157 u := url
158 if i := strings.Index(u, "://"); i >= 0 {
159 u = u[i+3:]
160 }
161 // Strip path.
162 if i := strings.Index(u, "/"); i >= 0 {
163 u = u[:i]
164 }
165 // Strip port.
166 if i := strings.LastIndex(u, ":"); i >= 0 {
167 u = u[:i]
168 }
169 return u
170 }
171
172 // logFetch appends a fetch record to the forage log.
173 func (f *Forager) logFetch(result *FetchResult) {
174 file, err := os.OpenFile(f.LogPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
175 if err != nil {
176 return
177 }
178 defer file.Close()
179 fmt.Fprintf(file, "- gen=%d url=%s status=%d size=%d hash=%s duration=%s\n",
180 f.gen, result.URL, result.StatusCode, len(result.Body),
181 result.Hash[:16], result.Duration.Round(time.Millisecond))
182 }
183
184 // BytesRemaining returns how many bytes can still be fetched this generation.
185 func (f *Forager) BytesRemaining() int64 {
186 f.mu.Lock()
187 defer f.mu.Unlock()
188 r := f.MaxBytesPerGen - f.bytesUsed
189 if r < 0 {
190 return 0
191 }
192 return r
193 }
194