compare.go raw

   1  package profile
   2  
   3  import (
   4  	"git.mleku.dev/mleku/dendrite/pkg/ratio"
   5  )
   6  
   7  // Verdict is the output of comparing a sample's stats against a baseline.
   8  type Verdict struct {
   9  	// HumanProbability estimates how likely the text is human-written.
  10  	// Range [0, 1] as a rational number.
  11  	HumanProbability ratio.Ratio `json:"human_probability"`
  12  
  13  	// Confidence measures how much signal the comparison found.
  14  	// Low confidence means the sample was too short or the baseline
  15  	// was insufficiently trained.
  16  	Confidence ratio.Ratio `json:"confidence"`
  17  
  18  	// Breakdown shows each metric's contribution to the verdict.
  19  	Breakdown map[string]ratio.Ratio `json:"breakdown"`
  20  
  21  	// ModelMatch is the name of the best-matching model fingerprint,
  22  	// if model comparison was performed. Empty if no model match.
  23  	ModelMatch string `json:"model_match,omitempty"`
  24  }
  25  
  26  // metricWeight defines the weight of a single metric in the comparison.
  27  type metricWeight struct {
  28  	name   string
  29  	weight ratio.Ratio
  30  }
  31  
  32  // defaultWeights are the initial metric weights for comparison.
  33  // Only scale-independent metrics are used: metrics that are already
  34  // normalized per-token or per-bond, so a 200-token sample can be
  35  // compared against a 2M-token baseline without distortion.
  36  //
  37  // Scale-dependent metrics (vertex_coverage, new_vertex_rate) are excluded
  38  // because they grow with sample size and produce false divergence when
  39  // comparing short samples against long baselines.
  40  var defaultWeights = []metricWeight{
  41  	{"bond_rate", ratio.New(25, 100)},          // bonds/tokens — does the text fit the lattice?
  42  	{"avg_walk_distance", ratio.New(25, 100)},  // steps/bond — how easily does it bond?
  43  	{"transition_entropy", ratio.New(25, 100)}, // bigram entropy — are transitions natural?
  44  	{"path_entropy", ratio.New(15, 100)},       // node entropy — diverse bonding?
  45  	{"surprisal_variance", ratio.New(10, 100)}, // surprisal spread — bursty or smooth?
  46  }
  47  
  48  // Compare compares a sample's stats against a baseline to produce a verdict.
  49  //
  50  // The comparison works by measuring how far each metric deviates from the
  51  // baseline. Human text on a human-trained lattice should produce stats close
  52  // to the baseline. AI text should deviate systematically: lower entropy,
  53  // lower surprisal variance, lower burstiness, etc.
  54  //
  55  // The deviation direction matters:
  56  //   - path_entropy: lower than baseline → more AI-like
  57  //   - surprisal_variance: lower → more AI-like
  58  //   - burstiness_gini: lower → more AI-like
  59  //   - vertex_coverage: deviation in either direction → suspicious
  60  //   - new_vertex_rate: higher → text contains patterns not in baseline
  61  //   - avg_walk_distance: higher → text doesn't fit the baseline well
  62  func Compare(baseline, sample Stats) Verdict {
  63  	v := Verdict{
  64  		Breakdown: make(map[string]ratio.Ratio, len(defaultWeights)),
  65  	}
  66  
  67  	// For each metric, compute a similarity score in [0, 1].
  68  	// 1 = identical to baseline, 0 = maximally divergent.
  69  	scores := make(map[string]ratio.Ratio, len(defaultWeights))
  70  
  71  	scores["bond_rate"] = similarity(baseline.BondRate, sample.BondRate)
  72  	scores["avg_walk_distance"] = similarity(baseline.AvgWalkDistance, sample.AvgWalkDistance)
  73  	scores["transition_entropy"] = similarity(baseline.TransitionEntropy, sample.TransitionEntropy)
  74  	scores["path_entropy"] = similarity(baseline.PathEntropy, sample.PathEntropy)
  75  	scores["surprisal_variance"] = similarity(baseline.SurprisalVariance, sample.SurprisalVariance)
  76  
  77  	// Weighted sum.
  78  	weightedSum := ratio.Zero
  79  	for _, mw := range defaultWeights {
  80  		score := scores[mw.name]
  81  		contribution := mw.weight.Mul(score)
  82  		v.Breakdown[mw.name] = score
  83  		weightedSum = weightedSum.Add(contribution)
  84  	}
  85  
  86  	v.HumanProbability = weightedSum.Clamp(ratio.Zero, ratio.One)
  87  
  88  	// Confidence is based on whether the baseline had enough data.
  89  	// More tokens in the baseline → higher confidence.
  90  	// This is a rough heuristic; calibration will refine it.
  91  	v.Confidence = ratio.One // placeholder until calibration
  92  
  93  	return v
  94  }
  95  
  96  // CompareModels compares a sample against multiple model fingerprints.
  97  // Returns the best-matching model name and its similarity score.
  98  func CompareModels(sample Stats, models map[string]Stats) (bestModel string, bestScore ratio.Ratio) {
  99  	bestScore = ratio.Zero
 100  	for name, modelStats := range models {
 101  		// For model matching, we want HIGH similarity to the model's profile.
 102  		score := modelSimilarity(sample, modelStats)
 103  		if score.Greater(bestScore) {
 104  			bestScore = score
 105  			bestModel = name
 106  		}
 107  	}
 108  	return
 109  }
 110  
 111  // similarity computes a similarity score between two metric values.
 112  // Returns a ratio in [0, 1] where 1 means identical.
 113  // Uses absolute relative difference: 1 - |a - b| / max(|a|, |b|, 1).
 114  func similarity(baseline, sample ratio.Ratio) ratio.Ratio {
 115  	diff := baseline.Sub(sample).Abs()
 116  	denom := ratio.Max(baseline.Abs(), sample.Abs())
 117  	denom = ratio.Max(denom, ratio.One) // avoid division by zero
 118  	relDiff := diff.Div(denom)
 119  	result := ratio.One.Sub(relDiff)
 120  	return result.Clamp(ratio.Zero, ratio.One)
 121  }
 122  
 123  // modelSimilarity computes overall similarity between sample and model stats.
 124  func modelSimilarity(sample, model Stats) ratio.Ratio {
 125  	total := ratio.Zero
 126  	n := ratio.Zero
 127  	total = total.Add(similarity(sample.PathEntropy, model.PathEntropy))
 128  	total = total.Add(similarity(sample.SurprisalVariance, model.SurprisalVariance))
 129  	total = total.Add(similarity(sample.BurstinessGini, model.BurstinessGini))
 130  	total = total.Add(similarity(sample.VertexCoverage, model.VertexCoverage))
 131  	total = total.Add(similarity(sample.AvgWalkDistance, model.AvgWalkDistance))
 132  	total = total.Add(similarity(sample.TransitionEntropy, model.TransitionEntropy))
 133  	n = ratio.FromInt(6)
 134  	return total.Div(n)
 135  }
 136