main.go raw

   1  // Command gutenberg downloads plain text books from Project Gutenberg
   2  // for use as a human text corpus in lattice-based text recognition.
   3  //
   4  // Usage:
   5  //
   6  //	gutenberg -output ./corpus -max 1000
   7  //	gutenberg -output ./corpus -max 500 -rate 3s
   8  package main
   9  
  10  import (
  11  	"context"
  12  	"flag"
  13  	"fmt"
  14  	"log"
  15  	"os"
  16  	"os/signal"
  17  	"time"
  18  
  19  	"git.mleku.dev/mleku/dendrite/pkg/gutenberg"
  20  )
  21  
  22  func main() {
  23  	var (
  24  		outputDir = flag.String("output", "./gutenberg_corpus", "output directory for downloaded texts")
  25  		maxBooks  = flag.Int("max", 100, "maximum number of books to download (0 = no limit)")
  26  		rateLimit = flag.Duration("rate", 2*time.Second, "delay between HTTP requests")
  27  		mirror    = flag.String("mirror", gutenberg.DefaultMirrorURL, "Gutenberg mirror base URL")
  28  	)
  29  
  30  	flag.Parse()
  31  
  32  	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
  33  	defer cancel()
  34  
  35  	s := &gutenberg.Scraper{
  36  		MirrorURL: *mirror,
  37  		OutputDir: *outputDir,
  38  		MaxBooks:  *maxBooks,
  39  		RateLimit: *rateLimit,
  40  	}
  41  
  42  	fmt.Printf("downloading up to %d books to %s\n", *maxBooks, *outputDir)
  43  	fmt.Printf("mirror: %s\n", *mirror)
  44  	fmt.Printf("rate limit: %s\n", *rateLimit)
  45  
  46  	if err := s.Run(ctx); err != nil {
  47  		log.Fatal(err)
  48  	}
  49  }
  50