langdesc.mx raw

   1  package iskra
   2  
   3  import "git.smesh.lol/iskradb/lattice"
   4  
   5  // Word order constants encoded in LangDesc.Order.
   6  const (
   7  	OrderSVO uint8 = 0
   8  	OrderSOV uint8 = 1
   9  	OrderVSO uint8 = 2
  10  	OrderVOS uint8 = 3
  11  	OrderOVS uint8 = 4
  12  	OrderOSV uint8 = 5
  13  )
  14  
  15  // Marker system constants encoded in LangDesc.Markers.
  16  const (
  17  	MarkerPrepositional  uint8 = 0
  18  	MarkerPostpositional uint8 = 1
  19  	MarkerCase           uint8 = 2
  20  )
  21  
  22  // LangDesc encodes the structural properties of a language for the cluster pipeline.
  23  // Stored in Bcooccur at MakeKey(domain, 0, "").
  24  type LangDesc struct {
  25  	Order      uint8 // word order: OrderSVO, OrderSOV, etc.
  26  	HeadFinal  bool  // false=head-initial (EN), true=head-final (JA)
  27  	Particle   bool  // false=position-bounded parser, true=particle-bounded
  28  	PreNomRC   bool  // false=post-nominal RC (EN), true=pre-nominal RC (JA)
  29  	ZeroCopula bool  // false=overt copula (EN), true=zero copula (JA)
  30  	Markers    uint8 // MarkerPrepositional / Postpositional / Case
  31  }
  32  
  33  // Particle role codes stored in Record.Branch of Bcooccur particle-role records.
  34  const (
  35  	RoleNone           uint8 = 0
  36  	RoleNPSubjTopic    uint8 = 1
  37  	RoleNPSubjGram     uint8 = 2
  38  	RoleNPObjDirect    uint8 = 3
  39  	RolePPSource       uint8 = 4
  40  	RolePPLimit        uint8 = 5
  41  	RolePPComitative   uint8 = 6
  42  	RolePPLocative     uint8 = 7
  43  	RolePPTemporal     uint8 = 8
  44  	RoleNPDative       uint8 = 9
  45  	RolePPLocStatic    uint8 = 10
  46  	RolePPInstrumental uint8 = 11
  47  )
  48  
  49  // CoordVerbClass is the sentinel coord used for verb class records in Bcooccur.
  50  // Distinct from particle role records (coord=0) and lang descriptors (coord=0, empty word).
  51  const CoordVerbClass uint64 = 1
  52  
  53  // KeyNormalizer is a per-domain hook that maps a raw word to its canonical
  54  // lookup form before hashing. nil = identity.
  55  //
  56  // Examples:
  57  //   LangEN: lowercase ("Cat" → "cat")
  58  //   LangJA: identity (surface form already canonical)
  59  //   DomainMoxieSRC: qualified name resolution ("Method" → "pkg.Type.Method")
  60  //   DomainMoxieIR: LLVM mangled name canonicalization
  61  type KeyNormalizer func(word string) string
  62  
  63  // Registry holds per-domain function registrations (key normalizers,
  64  // marker funcs, inflect funcs). Construct with NewRegistry(), populate
  65  // with Register* methods, query with Normalize*/Lookup* methods.
  66  type Registry struct {
  67  	keyNorm      map[uint8]KeyNormalizer
  68  	markerFn     map[uint8]MarkerFunc
  69  	inflectFn    map[uint8]InflectFunc
  70  	phrasalVerbs map[string]bool
  71  }
  72  
  73  // NewRegistry returns a ready-to-use Registry.
  74  func NewRegistry() *Registry {
  75  	return &Registry{
  76  		keyNorm:      map[uint8]KeyNormalizer{},
  77  		markerFn:     map[uint8]MarkerFunc{},
  78  		inflectFn:    map[uint8]InflectFunc{},
  79  		phrasalVerbs: map[string]bool{},
  80  	}
  81  }
  82  
  83  // RegisterKeyNormalizer stores a per-domain key normalization function.
  84  func (r *Registry) RegisterKeyNormalizer(domain uint8, fn KeyNormalizer) {
  85  	r.keyNorm[domain] = fn
  86  }
  87  
  88  // NormalizeKey applies the registered normalizer for domain, or returns word unchanged.
  89  func (r *Registry) NormalizeKey(domain uint8, word string) string {
  90  	if fn, ok := r.keyNorm[domain]; ok && fn != nil {
  91  		return fn(word)
  92  	}
  93  	return word
  94  }
  95  
  96  func packDescBranch(d LangDesc) uint8 {
  97  	v := d.Order & 0x7
  98  	if d.HeadFinal  { v |= 1 << 3 }
  99  	if d.Particle   { v |= 1 << 4 }
 100  	if d.PreNomRC   { v |= 1 << 5 }
 101  	if d.ZeroCopula { v |= 1 << 6 }
 102  	return v
 103  }
 104  
 105  func unpackDesc(b uint8, markerSys uint8) LangDesc {
 106  	return LangDesc{
 107  		Order:      b & 0x7,
 108  		HeadFinal:  (b>>3)&1 == 1,
 109  		Particle:   (b>>4)&1 == 1,
 110  		PreNomRC:   (b>>5)&1 == 1,
 111  		ZeroCopula: (b>>6)&1 == 1,
 112  		Markers:    markerSys & 0x3,
 113  	}
 114  }
 115  
 116  // RegisterLangDesc writes a LangDesc into Bcooccur at MakeKey(domain, 0, "").
 117  func RegisterLangDesc(tree *lattice.Tree, pool *[]byte, domain uint8, desc LangDesc) {
 118  	key := MakeKey(domain, 0, "")
 119  	ri := tree.LookupRecIdx(lattice.Bcooccur, key)
 120  	if ri != lattice.NullRec {
 121  		if r := tree.GetRecord(ri); r != nil {
 122  			r.Branch = packDescBranch(desc)
 123  			r.DataFile = uint32(desc.Markers&0x3) << 6
 124  		}
 125  		return
 126  	}
 127  	var rec lattice.Record
 128  	rec.Branch = packDescBranch(desc)
 129  	rec.DataFile = uint32(desc.Markers&0x3) << 6
 130  	tree.InsertRec(lattice.Bcooccur, key, rec)
 131  }
 132  
 133  // GetLangDesc reads a LangDesc from Bcooccur. Returns false if not registered.
 134  func GetLangDesc(tree *lattice.Tree, domain uint8) (LangDesc, bool) {
 135  	key := MakeKey(domain, 0, "")
 136  	ri := tree.LookupRecIdx(lattice.Bcooccur, key)
 137  	if ri == lattice.NullRec {
 138  		return LangDesc{}, false
 139  	}
 140  	rec := tree.GetRecord(ri)
 141  	if rec == nil {
 142  		return LangDesc{}, false
 143  	}
 144  	markerSys := uint8((rec.DataFile >> 6) & 0x3)
 145  	return unpackDesc(rec.Branch, markerSys), true
 146  }
 147  
 148  // RegisterParticleRole stores a particle→role mapping in Bcooccur.
 149  // semCoord=0 for unambiguous particles; semantic-coord for ambiguous disambiguation.
 150  func RegisterParticleRole(tree *lattice.Tree, pool *[]byte, domain uint8, semCoord uint64, particle string, role uint8) {
 151  	key := MakeKey(domain, semCoord, particle)
 152  	ri := tree.LookupRecIdx(lattice.Bcooccur, key)
 153  	if ri != lattice.NullRec {
 154  		if r := tree.GetRecord(ri); r != nil {
 155  			r.Branch = role
 156  		}
 157  		return
 158  	}
 159  	var rec lattice.Record
 160  	rec.Branch = role
 161  	tree.InsertRec(lattice.Bcooccur, key, rec)
 162  }
 163  
 164  // LookupParticleRole resolves a particle to its syntactic role using NP semantic flags.
 165  // Tries semantic-coord disambiguation first via RelaxCoord, then coord=0 default.
 166  func LookupParticleRole(tree *lattice.Tree, domain uint8, particle string, npFlags uint64) uint8 {
 167  	if npFlags != 0 {
 168  		semCoord := PackCoord(npFlags, 0, 0, 0, 0, 0, 0)
 169  		for _, c := range RelaxCoord(semCoord) {
 170  			if c == 0 {
 171  				break
 172  			}
 173  			ri := tree.LookupRecIdx(lattice.Bcooccur, MakeKey(domain, c, particle))
 174  			if ri != lattice.NullRec {
 175  				if rec := tree.GetRecord(ri); rec != nil && rec.Branch != 0 {
 176  					return rec.Branch
 177  				}
 178  			}
 179  		}
 180  	}
 181  	ri := tree.LookupRecIdx(lattice.Bcooccur, MakeKey(domain, 0, particle))
 182  	if ri != lattice.NullRec {
 183  		if rec := tree.GetRecord(ri); rec != nil {
 184  			return rec.Branch
 185  		}
 186  	}
 187  	return RoleNone
 188  }
 189  
 190  // MarkerFunc maps a particle role to its surface marker string for a given domain.
 191  type MarkerFunc func(role uint8) string
 192  
 193  // RegisterMarkerFunc registers the marker lookup function for a domain.
 194  func (r *Registry) RegisterMarkerFunc(domain uint8, fn MarkerFunc) {
 195  	r.markerFn[domain] = fn
 196  }
 197  
 198  // LookupTargetMarker returns the surface marker (preposition/postposition) for a
 199  // role in the target domain, calling the registered MarkerFunc.
 200  func (r *Registry) LookupTargetMarker(domain uint8, role uint8) string {
 201  	if fn, ok := r.markerFn[domain]; ok && fn != nil {
 202  		return fn(role)
 203  	}
 204  	return ""
 205  }
 206