// Package gutenberg scrapes plain text books from Project Gutenberg
// for use as an uncontaminated human text corpus. Targets the plain text
// format (.txt) which has the least formal artifacts.
//
// All downloaded texts are stripped of Gutenberg boilerplate headers
// and footers, leaving clean prose.
package gutenberg

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

// Scraper downloads and cleans Project Gutenberg texts.
type Scraper struct {
	// MirrorURL is the base URL for the Gutenberg mirror.
	// Default: https://www.gutenberg.org/cache/epub
	MirrorURL string

	// OutputDir is where cleaned text files are written.
	OutputDir string

	// MaxBooks limits how many books to download. 0 means no limit.
	MaxBooks int

	// Languages filters by ISO 639-1 language code. Empty means English only.
	Languages []string

	// RateLimit is the delay between HTTP requests. Default: 2 seconds.
	RateLimit time.Duration

	// Client is the HTTP client. If nil, a default with 30s timeout is used.
	Client *http.Client
}

// DefaultMirrorURL is the standard Gutenberg plain text location.
const DefaultMirrorURL = "https://www.gutenberg.org/cache/epub"

// Run downloads books sequentially, skipping already-downloaded ones.
// Book IDs are tried sequentially starting from 1.
func (s *Scraper) Run(ctx context.Context) error {
	if s.MirrorURL == "" {
		s.MirrorURL = DefaultMirrorURL
	}
	if s.RateLimit == 0 {
		s.RateLimit = 2 * time.Second
	}
	if s.Client == nil {
		s.Client = &http.Client{Timeout: 30 * time.Second}
	}
	if err := os.MkdirAll(s.OutputDir, 0o755); err != nil {
		return fmt.Errorf("create output dir: %w", err)
	}

	downloaded := 0
	failures := 0
	maxFailures := 500 // consecutive failures before giving up

	for bookID := 1; s.MaxBooks == 0 || downloaded < s.MaxBooks; bookID++ {
		select {
		case <-ctx.Done():
			return ctx.Err()
		default:
		}

		outPath := filepath.Join(s.OutputDir, fmt.Sprintf("%d.txt", bookID))

		// Skip if already downloaded.
		if _, err := os.Stat(outPath); err == nil {
			downloaded++
			failures = 0
			continue
		}

		// Try to download.
		text, err := s.downloadBook(ctx, bookID)
		if err != nil {
			failures++
			if failures > maxFailures {
				return fmt.Errorf("too many consecutive failures (%d), stopping at book %d", maxFailures, bookID)
			}
			continue
		}

		// Strip boilerplate.
		cleaned := StripBoilerplate(text)
		if len(cleaned) < 500 {
			// Too short after stripping — probably not a real book.
			continue
		}

		if err := os.WriteFile(outPath, []byte(cleaned), 0o644); err != nil {
			return fmt.Errorf("write %s: %w", outPath, err)
		}

		downloaded++
		failures = 0
		fmt.Printf("  [%d] book %d: %d bytes\n", downloaded, bookID, len(cleaned))

		// Rate limit.
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-time.After(s.RateLimit):
		}
	}

	fmt.Printf("downloaded %d books to %s\n", downloaded, s.OutputDir)
	return nil
}

// downloadBook fetches the plain text of a single book by ID.
func (s *Scraper) downloadBook(ctx context.Context, bookID int) (string, error) {
	// Try the standard plain text URL pattern.
	url := fmt.Sprintf("%s/%d/pg%d.txt", s.MirrorURL, bookID, bookID)

	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
	if err != nil {
		return "", err
	}
	req.Header.Set("User-Agent", "dendrite-gutenberg-scraper/1.0 (text-recognition research)")

	resp, err := s.Client.Do(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf("HTTP %d for book %d", resp.StatusCode, bookID)
	}

	// Limit read to 10MB to avoid memory issues with huge books.
	body, err := io.ReadAll(io.LimitReader(resp.Body, 10<<20))
	if err != nil {
		return "", err
	}

	return string(body), nil
}

// StripBoilerplate removes Project Gutenberg headers and footers from text.
//
// Gutenberg texts use standard markers:
//   - "*** START OF THE PROJECT GUTENBERG EBOOK" (or variations)
//   - "*** END OF THE PROJECT GUTENBERG EBOOK" (or variations)
//
// Everything before the start marker and after the end marker is removed.
func StripBoilerplate(text string) string {
	scanner := bufio.NewScanner(strings.NewReader(text))
	var lines []string
	inBody := false

	for scanner.Scan() {
		line := scanner.Text()
		upper := strings.ToUpper(line)

		if !inBody {
			// Look for start marker.
			if strings.Contains(upper, "*** START OF") ||
				strings.Contains(upper, "***START OF") {
				inBody = true
				continue
			}
			continue
		}

		// Look for end marker.
		if strings.Contains(upper, "*** END OF") ||
			strings.Contains(upper, "***END OF") {
			break
		}

		lines = append(lines, line)
	}

	// If no markers found, return the original text (some older texts
	// don't have the standard markers).
	if len(lines) == 0 {
		return text
	}

	return strings.Join(lines, "\n")
}

// BookIDsFromCatalog parses a list of book IDs from a Gutenberg catalog file.
// The catalog is a simple text file with one ID per line.
func BookIDsFromCatalog(r io.Reader) ([]int, error) {
	scanner := bufio.NewScanner(r)
	var ids []int
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}
		id, err := strconv.Atoi(line)
		if err != nil {
			continue
		}
		ids = append(ids, id)
	}
	return ids, scanner.Err()
}