// Package forage — search.go provides search integration for autonomous // text discovery. Two sources: Archive.org and Gutendex (Project Gutenberg). package forage import ( "context" "encoding/json" "fmt" "io" "math/rand/v2" "net/http" "net/url" "strings" "time" ) // SearchResult represents a single text found by a search source. type SearchResult struct { Title string // human-readable title URL string // direct download URL for plain text Source string // "archive.org" or "gutenberg" } // SearchArchiveOrg queries archive.org's advanced search for text-mediatype // items matching the query. Returns up to 5 results sorted by popularity. func SearchArchiveOrg(ctx context.Context, query string) ([]SearchResult, error) { q := url.Values{} q.Set("q", query+" mediatype:texts") q.Set("fl[]", "identifier,title") q.Set("sort[]", "downloads desc") q.Set("rows", "5") q.Set("output", "json") reqURL := "https://archive.org/advancedsearch.php?" + q.Encode() req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) if err != nil { return nil, err } req.Header.Set("User-Agent", "dendrite-curiosity/0.1") resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != 200 { return nil, fmt.Errorf("archive.org: status %d", resp.StatusCode) } body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) if err != nil { return nil, err } var result struct { Response struct { Docs []struct { Identifier string `json:"identifier"` Title string `json:"title"` } `json:"docs"` } `json:"response"` } if err := json.Unmarshal(body, &result); err != nil { return nil, fmt.Errorf("archive.org: %w", err) } var results []SearchResult for _, doc := range result.Response.Docs { if doc.Identifier == "" { continue } results = append(results, SearchResult{ Title: doc.Title, URL: fmt.Sprintf("https://archive.org/download/%s/%s_djvu.txt", doc.Identifier, doc.Identifier), Source: "archive.org", }) } return results, nil } // SearchGutendex queries the Gutendex API for English-language books // matching the query. Returns up to 5 results with plain-text download URLs. func SearchGutendex(ctx context.Context, query string) ([]SearchResult, error) { q := url.Values{} q.Set("search", query) q.Set("languages", "en") reqURL := "https://gutendex.com/books?" + q.Encode() req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) if err != nil { return nil, err } req.Header.Set("User-Agent", "dendrite-curiosity/0.1") resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != 200 { return nil, fmt.Errorf("gutendex: status %d", resp.StatusCode) } body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) if err != nil { return nil, err } var result struct { Results []struct { Title string `json:"title"` Formats map[string]string `json:"formats"` } `json:"results"` } if err := json.Unmarshal(body, &result); err != nil { return nil, fmt.Errorf("gutendex: %w", err) } var results []SearchResult for _, book := range result.Results { if len(results) >= 5 { break } // Look for plain text format. textURL := "" for mime, u := range book.Formats { if strings.Contains(mime, "text/plain") && !strings.Contains(mime, "zip") { textURL = u break } } if textURL == "" { continue } results = append(results, SearchResult{ Title: book.Title, URL: textURL, Source: "gutenberg", }) } return results, nil } // RandomGutenbergURL returns a URL for a random Project Gutenberg text. // Used as fallback when baby talk queries produce no search results. func RandomGutenbergURL(seed uint64) SearchResult { rng := rand.New(rand.NewPCG(seed, seed^0xfeed)) // Gutenberg has ~70000 books. IDs 1-70000. id := rng.IntN(70000) + 1 return SearchResult{ Title: fmt.Sprintf("Gutenberg #%d", id), URL: fmt.Sprintf("https://www.gutenberg.org/cache/epub/%d/pg%d.txt", id, id), Source: "gutenberg-random", } } // SearchAll queries both Archive.org and Gutendex, merging results. // Archive.org results come first (broader catalog), then Gutenberg. // Returns at most 10 results total. func SearchAll(ctx context.Context, query string) []SearchResult { type resultSet struct { results []SearchResult err error } archiveCh := make(chan resultSet, 1) gutendexCh := make(chan resultSet, 1) go func() { r, err := SearchArchiveOrg(ctx, query) archiveCh <- resultSet{r, err} }() go func() { r, err := SearchGutendex(ctx, query) gutendexCh <- resultSet{r, err} }() var all []SearchResult ar := <-archiveCh if ar.err == nil { all = append(all, ar.results...) } gr := <-gutendexCh if gr.err == nil { all = append(all, gr.results...) } if len(all) > 10 { all = all[:10] } return all } // PickBestResult returns the first search result whose URL hasn't been seen. // Returns nil if all results have been seen. func PickBestResult(results []SearchResult, seenURLs map[string]bool) *SearchResult { for i := range results { if !seenURLs[results[i].URL] { return &results[i] } } return nil }