// Command gutenberg downloads plain text books from Project Gutenberg
// for use as a human text corpus in lattice-based text recognition.
//
// Usage:
//
//	gutenberg -output ./corpus -max 1000
//	gutenberg -output ./corpus -max 500 -rate 3s
package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"time"

	"git.mleku.dev/mleku/dendrite/pkg/gutenberg"
)

func main() {
	var (
		outputDir = flag.String("output", "./gutenberg_corpus", "output directory for downloaded texts")
		maxBooks  = flag.Int("max", 100, "maximum number of books to download (0 = no limit)")
		rateLimit = flag.Duration("rate", 2*time.Second, "delay between HTTP requests")
		mirror    = flag.String("mirror", gutenberg.DefaultMirrorURL, "Gutenberg mirror base URL")
	)

	flag.Parse()

	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
	defer cancel()

	s := &gutenberg.Scraper{
		MirrorURL: *mirror,
		OutputDir: *outputDir,
		MaxBooks:  *maxBooks,
		RateLimit: *rateLimit,
	}

	fmt.Printf("downloading up to %d books to %s\n", *maxBooks, *outputDir)
	fmt.Printf("mirror: %s\n", *mirror)
	fmt.Printf("rate limit: %s\n", *rateLimit)

	if err := s.Run(ctx); err != nil {
		log.Fatal(err)
	}
}