text.go raw

   1  package grammar
   2  
   3  import (
   4  	"git.mleku.dev/mleku/dendrite/pkg/ratio"
   5  )
   6  
   7  // NaturalText is the grammar for natural language text recognition.
   8  //
   9  // Word tokens are sub-classified by length bucket (w1..w5) to create a
  10  // richer constraint envelope. The adjacency rules encode which word length
  11  // classes can neighbor each other, punctuation, and spaces. The lattice
  12  // topology shaped by these rules captures word-length transition patterns
  13  // that differ between human and AI text.
  14  //
  15  // Word length buckets:
  16  //
  17  //	w1: 1 char      (a, I)
  18  //	w2: 2-3 chars   (the, is, an)
  19  //	w3: 4-5 chars   (from, with, about)
  20  //	w4: 6-8 chars   (between, another)
  21  //	w5: 9+ chars    (restructured, acknowledging)
  22  var NaturalText = &Grammar{Rules: []Rule{
  23  	// Short words connect to everything — they're the glue of English.
  24  	{Tag: "w1", Neighbors: []string{"w1", "w2", "w3", "w4", "w5", "punct", "space"}},
  25  	{Tag: "w2", Neighbors: []string{"w1", "w2", "w3", "w4", "w5", "punct", "space"}},
  26  	// Medium words: most flexible.
  27  	{Tag: "w3", Neighbors: []string{"w1", "w2", "w3", "w4", "w5", "punct", "space"}},
  28  	// Longer words tend to precede short connectors or punctuation.
  29  	{Tag: "w4", Neighbors: []string{"w1", "w2", "w3", "w4", "w5", "punct", "space"}},
  30  	{Tag: "w5", Neighbors: []string{"w1", "w2", "w3", "w4", "punct", "space"}},
  31  	// Punctuation bridges words and other punctuation.
  32  	{Tag: "punct", Neighbors: []string{"w1", "w2", "w3", "w4", "w5", "punct", "space"}},
  33  	// Spaces always lead to words.
  34  	{Tag: "space", Neighbors: []string{"w1", "w2", "w3", "w4", "w5", "punct"}},
  35  }}
  36  
  37  // TextDefaultCounts returns a tag count map for building a natural language
  38  // lattice. Distributes targetSize nodes across the seven text tags using
  39  // proportions derived from typical English prose word-length distributions:
  40  //
  41  //	w1: ~5%   (single-char words are rare)
  42  //	w2: ~20%  (very common: the, is, an, to, of)
  43  //	w3: ~20%  (common: from, with, that, about)
  44  //	w4: ~15%  (content words: between, another)
  45  //	w5: ~5%   (long formal words)
  46  //	punct: ~10%
  47  //	space: ~25%
  48  func TextDefaultCounts(targetSize int) map[string]int {
  49  	if targetSize < 7 {
  50  		targetSize = 7
  51  	}
  52  	w1 := int(ratio.New(5, 100).ScaleInt(int64(targetSize)))
  53  	w2 := int(ratio.New(20, 100).ScaleInt(int64(targetSize)))
  54  	w3 := int(ratio.New(20, 100).ScaleInt(int64(targetSize)))
  55  	w4 := int(ratio.New(15, 100).ScaleInt(int64(targetSize)))
  56  	w5 := int(ratio.New(5, 100).ScaleInt(int64(targetSize)))
  57  	punct := int(ratio.New(10, 100).ScaleInt(int64(targetSize)))
  58  	space := targetSize - w1 - w2 - w3 - w4 - w5 - punct
  59  
  60  	// Ensure each type has at least 1 node.
  61  	counts := map[string]int{
  62  		"w1": w1, "w2": w2, "w3": w3, "w4": w4, "w5": w5,
  63  		"punct": punct, "space": space,
  64  	}
  65  	for k, v := range counts {
  66  		if v < 1 {
  67  			counts[k] = 1
  68  		}
  69  	}
  70  	return counts
  71  }
  72