text_test.go raw

   1  package grammar
   2  
   3  import (
   4  	"testing"
   5  
   6  	"git.mleku.dev/mleku/dendrite/pkg/axiom"
   7  )
   8  
   9  func TestNaturalTextAdjacency(t *testing.T) {
  10  	g := NaturalText
  11  
  12  	tests := []struct {
  13  		a, b string
  14  		want bool
  15  	}{
  16  		// Word length classes neighbor each other and punct/space.
  17  		{"w2", "w3", true},
  18  		{"w3", "punct", true},
  19  		{"w1", "space", true},
  20  		{"punct", "w2", true},
  21  		{"punct", "punct", true},
  22  		{"punct", "space", true},
  23  		{"space", "w3", true},
  24  		{"space", "punct", true},
  25  		{"space", "space", false}, // spaces don't neighbor spaces
  26  		// w5 doesn't neighbor w5 (long words rarely follow long words).
  27  		{"w5", "w5", false},
  28  	}
  29  
  30  	for _, tt := range tests {
  31  		if got := g.CanNeighbor(tt.a, tt.b); got != tt.want {
  32  			t.Errorf("CanNeighbor(%q, %q) = %v, want %v", tt.a, tt.b, got, tt.want)
  33  		}
  34  	}
  35  }
  36  
  37  func TestNaturalTextTags(t *testing.T) {
  38  	tags := NaturalText.Tags()
  39  	if len(tags) != 7 {
  40  		t.Errorf("got %d tags, want 7: %v", len(tags), tags)
  41  	}
  42  	expected := map[string]bool{
  43  		"w1": true, "w2": true, "w3": true, "w4": true, "w5": true,
  44  		"punct": true, "space": true,
  45  	}
  46  	for _, tag := range tags {
  47  		if !expected[tag] {
  48  			t.Errorf("unexpected tag: %s", tag)
  49  		}
  50  	}
  51  }
  52  
  53  func TestTextDefaultCounts(t *testing.T) {
  54  	counts := TextDefaultCounts(1000)
  55  
  56  	total := 0
  57  	for _, v := range counts {
  58  		total += v
  59  	}
  60  	if total != 1000 {
  61  		t.Errorf("total = %d, want 1000", total)
  62  	}
  63  
  64  	// Word classes combined should be ~65% (5+20+20+15+5).
  65  	wordTotal := counts["w1"] + counts["w2"] + counts["w3"] + counts["w4"] + counts["w5"]
  66  	if wordTotal < 500 || wordTotal > 750 {
  67  		t.Errorf("word total = %d, expected ~650", wordTotal)
  68  	}
  69  }
  70  
  71  func TestTextDefaultCountsMinimum(t *testing.T) {
  72  	counts := TextDefaultCounts(1)
  73  	for tag, n := range counts {
  74  		if n < 1 {
  75  			t.Errorf("%s count = %d, want >= 1", tag, n)
  76  		}
  77  	}
  78  }
  79  
  80  func TestBuildTextLattice(t *testing.T) {
  81  	counts := TextDefaultCounts(100)
  82  	seed := [32]byte{1, 2, 3}
  83  
  84  	l := BuildGrammarLattice(NaturalText, counts, seed, func(tag string) axiom.Constraint {
  85  		return NewConstraint(tag, NaturalText)
  86  	})
  87  
  88  	if l.Size() != 100 {
  89  		t.Errorf("lattice size = %d, want 100", l.Size())
  90  	}
  91  
  92  	// All nodes should have at least one neighbor (ring connectivity).
  93  	for _, n := range l.Nodes() {
  94  		if len(n.Neighbors()) == 0 {
  95  			t.Errorf("node %d has no neighbors", n.ID())
  96  		}
  97  	}
  98  }
  99