search.go raw
1 // Package forage — search.go provides search integration for autonomous
2 // text discovery. Two sources: Archive.org and Gutendex (Project Gutenberg).
3 package forage
4
5 import (
6 "context"
7 "encoding/json"
8 "fmt"
9 "io"
10 "math/rand/v2"
11 "net/http"
12 "net/url"
13 "strings"
14 "time"
15 )
16
17 // SearchResult represents a single text found by a search source.
18 type SearchResult struct {
19 Title string // human-readable title
20 URL string // direct download URL for plain text
21 Source string // "archive.org" or "gutenberg"
22 }
23
24 // SearchArchiveOrg queries archive.org's advanced search for text-mediatype
25 // items matching the query. Returns up to 5 results sorted by popularity.
26 func SearchArchiveOrg(ctx context.Context, query string) ([]SearchResult, error) {
27 q := url.Values{}
28 q.Set("q", query+" mediatype:texts")
29 q.Set("fl[]", "identifier,title")
30 q.Set("sort[]", "downloads desc")
31 q.Set("rows", "5")
32 q.Set("output", "json")
33
34 reqURL := "https://archive.org/advancedsearch.php?" + q.Encode()
35
36 req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
37 if err != nil {
38 return nil, err
39 }
40 req.Header.Set("User-Agent", "dendrite-curiosity/0.1")
41
42 resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req)
43 if err != nil {
44 return nil, err
45 }
46 defer resp.Body.Close()
47
48 if resp.StatusCode != 200 {
49 return nil, fmt.Errorf("archive.org: status %d", resp.StatusCode)
50 }
51
52 body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
53 if err != nil {
54 return nil, err
55 }
56
57 var result struct {
58 Response struct {
59 Docs []struct {
60 Identifier string `json:"identifier"`
61 Title string `json:"title"`
62 } `json:"docs"`
63 } `json:"response"`
64 }
65 if err := json.Unmarshal(body, &result); err != nil {
66 return nil, fmt.Errorf("archive.org: %w", err)
67 }
68
69 var results []SearchResult
70 for _, doc := range result.Response.Docs {
71 if doc.Identifier == "" {
72 continue
73 }
74 results = append(results, SearchResult{
75 Title: doc.Title,
76 URL: fmt.Sprintf("https://archive.org/download/%s/%s_djvu.txt", doc.Identifier, doc.Identifier),
77 Source: "archive.org",
78 })
79 }
80 return results, nil
81 }
82
83 // SearchGutendex queries the Gutendex API for English-language books
84 // matching the query. Returns up to 5 results with plain-text download URLs.
85 func SearchGutendex(ctx context.Context, query string) ([]SearchResult, error) {
86 q := url.Values{}
87 q.Set("search", query)
88 q.Set("languages", "en")
89
90 reqURL := "https://gutendex.com/books?" + q.Encode()
91
92 req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
93 if err != nil {
94 return nil, err
95 }
96 req.Header.Set("User-Agent", "dendrite-curiosity/0.1")
97
98 resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req)
99 if err != nil {
100 return nil, err
101 }
102 defer resp.Body.Close()
103
104 if resp.StatusCode != 200 {
105 return nil, fmt.Errorf("gutendex: status %d", resp.StatusCode)
106 }
107
108 body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
109 if err != nil {
110 return nil, err
111 }
112
113 var result struct {
114 Results []struct {
115 Title string `json:"title"`
116 Formats map[string]string `json:"formats"`
117 } `json:"results"`
118 }
119 if err := json.Unmarshal(body, &result); err != nil {
120 return nil, fmt.Errorf("gutendex: %w", err)
121 }
122
123 var results []SearchResult
124 for _, book := range result.Results {
125 if len(results) >= 5 {
126 break
127 }
128 // Look for plain text format.
129 textURL := ""
130 for mime, u := range book.Formats {
131 if strings.Contains(mime, "text/plain") && !strings.Contains(mime, "zip") {
132 textURL = u
133 break
134 }
135 }
136 if textURL == "" {
137 continue
138 }
139 results = append(results, SearchResult{
140 Title: book.Title,
141 URL: textURL,
142 Source: "gutenberg",
143 })
144 }
145 return results, nil
146 }
147
148 // RandomGutenbergURL returns a URL for a random Project Gutenberg text.
149 // Used as fallback when baby talk queries produce no search results.
150 func RandomGutenbergURL(seed uint64) SearchResult {
151 rng := rand.New(rand.NewPCG(seed, seed^0xfeed))
152 // Gutenberg has ~70000 books. IDs 1-70000.
153 id := rng.IntN(70000) + 1
154 return SearchResult{
155 Title: fmt.Sprintf("Gutenberg #%d", id),
156 URL: fmt.Sprintf("https://www.gutenberg.org/cache/epub/%d/pg%d.txt", id, id),
157 Source: "gutenberg-random",
158 }
159 }
160
161 // SearchAll queries both Archive.org and Gutendex, merging results.
162 // Archive.org results come first (broader catalog), then Gutenberg.
163 // Returns at most 10 results total.
164 func SearchAll(ctx context.Context, query string) []SearchResult {
165 type resultSet struct {
166 results []SearchResult
167 err error
168 }
169
170 archiveCh := make(chan resultSet, 1)
171 gutendexCh := make(chan resultSet, 1)
172
173 go func() {
174 r, err := SearchArchiveOrg(ctx, query)
175 archiveCh <- resultSet{r, err}
176 }()
177 go func() {
178 r, err := SearchGutendex(ctx, query)
179 gutendexCh <- resultSet{r, err}
180 }()
181
182 var all []SearchResult
183
184 ar := <-archiveCh
185 if ar.err == nil {
186 all = append(all, ar.results...)
187 }
188
189 gr := <-gutendexCh
190 if gr.err == nil {
191 all = append(all, gr.results...)
192 }
193
194 if len(all) > 10 {
195 all = all[:10]
196 }
197 return all
198 }
199
200 // PickBestResult returns the first search result whose URL hasn't been seen.
201 // Returns nil if all results have been seen.
202 func PickBestResult(results []SearchResult, seenURLs map[string]bool) *SearchResult {
203 for i := range results {
204 if !seenURLs[results[i].URL] {
205 return &results[i]
206 }
207 }
208 return nil
209 }
210