scraper.go raw
1 // Package gutenberg scrapes plain text books from Project Gutenberg
2 // for use as an uncontaminated human text corpus. Targets the plain text
3 // format (.txt) which has the least formal artifacts.
4 //
5 // All downloaded texts are stripped of Gutenberg boilerplate headers
6 // and footers, leaving clean prose.
7 package gutenberg
8
9 import (
10 "bufio"
11 "context"
12 "fmt"
13 "io"
14 "net/http"
15 "os"
16 "path/filepath"
17 "strconv"
18 "strings"
19 "time"
20 )
21
22 // Scraper downloads and cleans Project Gutenberg texts.
23 type Scraper struct {
24 // MirrorURL is the base URL for the Gutenberg mirror.
25 // Default: https://www.gutenberg.org/cache/epub
26 MirrorURL string
27
28 // OutputDir is where cleaned text files are written.
29 OutputDir string
30
31 // MaxBooks limits how many books to download. 0 means no limit.
32 MaxBooks int
33
34 // Languages filters by ISO 639-1 language code. Empty means English only.
35 Languages []string
36
37 // RateLimit is the delay between HTTP requests. Default: 2 seconds.
38 RateLimit time.Duration
39
40 // Client is the HTTP client. If nil, a default with 30s timeout is used.
41 Client *http.Client
42 }
43
44 // DefaultMirrorURL is the standard Gutenberg plain text location.
45 const DefaultMirrorURL = "https://www.gutenberg.org/cache/epub"
46
47 // Run downloads books sequentially, skipping already-downloaded ones.
48 // Book IDs are tried sequentially starting from 1.
49 func (s *Scraper) Run(ctx context.Context) error {
50 if s.MirrorURL == "" {
51 s.MirrorURL = DefaultMirrorURL
52 }
53 if s.RateLimit == 0 {
54 s.RateLimit = 2 * time.Second
55 }
56 if s.Client == nil {
57 s.Client = &http.Client{Timeout: 30 * time.Second}
58 }
59 if err := os.MkdirAll(s.OutputDir, 0o755); err != nil {
60 return fmt.Errorf("create output dir: %w", err)
61 }
62
63 downloaded := 0
64 failures := 0
65 maxFailures := 500 // consecutive failures before giving up
66
67 for bookID := 1; s.MaxBooks == 0 || downloaded < s.MaxBooks; bookID++ {
68 select {
69 case <-ctx.Done():
70 return ctx.Err()
71 default:
72 }
73
74 outPath := filepath.Join(s.OutputDir, fmt.Sprintf("%d.txt", bookID))
75
76 // Skip if already downloaded.
77 if _, err := os.Stat(outPath); err == nil {
78 downloaded++
79 failures = 0
80 continue
81 }
82
83 // Try to download.
84 text, err := s.downloadBook(ctx, bookID)
85 if err != nil {
86 failures++
87 if failures > maxFailures {
88 return fmt.Errorf("too many consecutive failures (%d), stopping at book %d", maxFailures, bookID)
89 }
90 continue
91 }
92
93 // Strip boilerplate.
94 cleaned := StripBoilerplate(text)
95 if len(cleaned) < 500 {
96 // Too short after stripping — probably not a real book.
97 continue
98 }
99
100 if err := os.WriteFile(outPath, []byte(cleaned), 0o644); err != nil {
101 return fmt.Errorf("write %s: %w", outPath, err)
102 }
103
104 downloaded++
105 failures = 0
106 fmt.Printf(" [%d] book %d: %d bytes\n", downloaded, bookID, len(cleaned))
107
108 // Rate limit.
109 select {
110 case <-ctx.Done():
111 return ctx.Err()
112 case <-time.After(s.RateLimit):
113 }
114 }
115
116 fmt.Printf("downloaded %d books to %s\n", downloaded, s.OutputDir)
117 return nil
118 }
119
120 // downloadBook fetches the plain text of a single book by ID.
121 func (s *Scraper) downloadBook(ctx context.Context, bookID int) (string, error) {
122 // Try the standard plain text URL pattern.
123 url := fmt.Sprintf("%s/%d/pg%d.txt", s.MirrorURL, bookID, bookID)
124
125 req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
126 if err != nil {
127 return "", err
128 }
129 req.Header.Set("User-Agent", "dendrite-gutenberg-scraper/1.0 (text-recognition research)")
130
131 resp, err := s.Client.Do(req)
132 if err != nil {
133 return "", err
134 }
135 defer resp.Body.Close()
136
137 if resp.StatusCode != http.StatusOK {
138 return "", fmt.Errorf("HTTP %d for book %d", resp.StatusCode, bookID)
139 }
140
141 // Limit read to 10MB to avoid memory issues with huge books.
142 body, err := io.ReadAll(io.LimitReader(resp.Body, 10<<20))
143 if err != nil {
144 return "", err
145 }
146
147 return string(body), nil
148 }
149
150 // StripBoilerplate removes Project Gutenberg headers and footers from text.
151 //
152 // Gutenberg texts use standard markers:
153 // - "*** START OF THE PROJECT GUTENBERG EBOOK" (or variations)
154 // - "*** END OF THE PROJECT GUTENBERG EBOOK" (or variations)
155 //
156 // Everything before the start marker and after the end marker is removed.
157 func StripBoilerplate(text string) string {
158 scanner := bufio.NewScanner(strings.NewReader(text))
159 var lines []string
160 inBody := false
161
162 for scanner.Scan() {
163 line := scanner.Text()
164 upper := strings.ToUpper(line)
165
166 if !inBody {
167 // Look for start marker.
168 if strings.Contains(upper, "*** START OF") ||
169 strings.Contains(upper, "***START OF") {
170 inBody = true
171 continue
172 }
173 continue
174 }
175
176 // Look for end marker.
177 if strings.Contains(upper, "*** END OF") ||
178 strings.Contains(upper, "***END OF") {
179 break
180 }
181
182 lines = append(lines, line)
183 }
184
185 // If no markers found, return the original text (some older texts
186 // don't have the standard markers).
187 if len(lines) == 0 {
188 return text
189 }
190
191 return strings.Join(lines, "\n")
192 }
193
194 // BookIDsFromCatalog parses a list of book IDs from a Gutenberg catalog file.
195 // The catalog is a simple text file with one ID per line.
196 func BookIDsFromCatalog(r io.Reader) ([]int, error) {
197 scanner := bufio.NewScanner(r)
198 var ids []int
199 for scanner.Scan() {
200 line := strings.TrimSpace(scanner.Text())
201 if line == "" || strings.HasPrefix(line, "#") {
202 continue
203 }
204 id, err := strconv.Atoi(line)
205 if err != nil {
206 continue
207 }
208 ids = append(ids, id)
209 }
210 return ids, scanner.Err()
211 }
212