scraper_test.go raw

   1  package gutenberg
   2  
   3  import (
   4  	"strings"
   5  	"testing"
   6  )
   7  
   8  func TestStripBoilerplate(t *testing.T) {
   9  	input := `The Project Gutenberg eBook of Test Book
  10  
  11  *** START OF THE PROJECT GUTENBERG EBOOK TEST BOOK ***
  12  
  13  This is the actual content of the book.
  14  It has multiple lines.
  15  And some more content here.
  16  
  17  *** END OF THE PROJECT GUTENBERG EBOOK TEST BOOK ***
  18  
  19  End of the Project Gutenberg EBook of Test Book
  20  Blah blah license info.`
  21  
  22  	result := StripBoilerplate(input)
  23  
  24  	if strings.Contains(result, "Project Gutenberg") {
  25  		t.Error("boilerplate not fully stripped")
  26  	}
  27  	if !strings.Contains(result, "actual content") {
  28  		t.Error("body content was stripped")
  29  	}
  30  	if !strings.Contains(result, "multiple lines") {
  31  		t.Error("body content was stripped")
  32  	}
  33  	if strings.Contains(result, "license info") {
  34  		t.Error("footer not stripped")
  35  	}
  36  }
  37  
  38  func TestStripBoilerplateNoMarkers(t *testing.T) {
  39  	input := "This is plain text without any Gutenberg markers."
  40  	result := StripBoilerplate(input)
  41  	if result != input {
  42  		t.Error("text without markers should be returned unchanged")
  43  	}
  44  }
  45  
  46  func TestStripBoilerplateCaseInsensitive(t *testing.T) {
  47  	input := `header stuff
  48  *** start of THE PROJECT GUTENBERG EBOOK ***
  49  body content
  50  *** END of THE PROJECT GUTENBERG EBOOK ***
  51  footer`
  52  
  53  	result := StripBoilerplate(input)
  54  	if !strings.Contains(result, "body content") {
  55  		t.Error("case-insensitive matching failed")
  56  	}
  57  }
  58  
  59  func TestBookIDsFromCatalog(t *testing.T) {
  60  	input := strings.NewReader(`# Comment
  61  1
  62  2
  63  3
  64  # Another comment
  65  
  66  5
  67  not-a-number
  68  10
  69  `)
  70  
  71  	ids, err := BookIDsFromCatalog(input)
  72  	if err != nil {
  73  		t.Fatal(err)
  74  	}
  75  	if len(ids) != 5 {
  76  		t.Errorf("got %d IDs, want 5", len(ids))
  77  	}
  78  	expected := []int{1, 2, 3, 5, 10}
  79  	for i, want := range expected {
  80  		if ids[i] != want {
  81  			t.Errorf("ids[%d] = %d, want %d", i, ids[i], want)
  82  		}
  83  	}
  84  }
  85