digest.go raw

   1  package forage
   2  
   3  import (
   4  	"strings"
   5  
   6  	"git.mleku.dev/mleku/dendrite/pkg/axiom"
   7  	"git.mleku.dev/mleku/dendrite/pkg/enzyme"
   8  )
   9  
  10  // Digest routes fetched content through the appropriate enzyme based
  11  // on content type and URL. Returns typed lattice elements.
  12  func Digest(result *FetchResult) []axiom.Element {
  13  	if result.StatusCode != 200 || len(result.Body) == 0 {
  14  		return nil
  15  	}
  16  
  17  	ct := strings.ToLower(result.ContentType)
  18  
  19  	switch {
  20  	case strings.Contains(ct, "text/html"):
  21  		return digestHTML(result.Body)
  22  
  23  	case strings.Contains(ct, "text/markdown"),
  24  		strings.HasSuffix(result.URL, ".md"):
  25  		return digestMarkdown(result.Body)
  26  
  27  	case strings.Contains(ct, "text/plain"),
  28  		strings.Contains(ct, "application/json"):
  29  		return digestText(result.Body)
  30  
  31  	case strings.HasSuffix(result.URL, ".go"):
  32  		return digestGoSource(result.Body)
  33  
  34  	default:
  35  		return digestText(result.Body)
  36  	}
  37  }
  38  
  39  // digestMarkdown decomposes markdown into typed elements.
  40  // Headers become "heading" elements, code blocks become "code" elements,
  41  // prose becomes "word" elements (same as text enzyme).
  42  func digestMarkdown(body []byte) []axiom.Element {
  43  	var elems []axiom.Element
  44  	lines := strings.Split(string(body), "\n")
  45  	inCodeBlock := false
  46  
  47  	for _, line := range lines {
  48  		trimmed := strings.TrimSpace(line)
  49  
  50  		if strings.HasPrefix(trimmed, "```") {
  51  			inCodeBlock = !inCodeBlock
  52  			continue
  53  		}
  54  
  55  		if inCodeBlock {
  56  			if trimmed != "" {
  57  				elems = append(elems, enzyme.HexElem("code", trimmed))
  58  			}
  59  			continue
  60  		}
  61  
  62  		if trimmed == "" {
  63  			continue
  64  		}
  65  
  66  		// Headings.
  67  		if strings.HasPrefix(trimmed, "#") {
  68  			heading := strings.TrimLeft(trimmed, "# ")
  69  			if heading != "" {
  70  				elems = append(elems, enzyme.HexElem("heading", heading))
  71  			}
  72  			continue
  73  		}
  74  
  75  		// Extract words from prose.
  76  		elems = append(elems, extractWords(trimmed)...)
  77  	}
  78  
  79  	return elems
  80  }
  81  
  82  // digestHTML strips tags and decomposes the text content.
  83  func digestHTML(body []byte) []axiom.Element {
  84  	// Simple HTML stripping — no dependency on a full parser.
  85  	text := stripHTMLTags(string(body))
  86  	return digestText([]byte(text))
  87  }
  88  
  89  // digestText decomposes plain text into word elements using the
  90  // text enzyme pattern.
  91  func digestText(body []byte) []axiom.Element {
  92  	var elems []axiom.Element
  93  	ch := enzyme.Text{}.Digest(strings.NewReader(string(body)))
  94  	for e := range ch {
  95  		if e.Type() == "space" {
  96  			continue
  97  		}
  98  		elems = append(elems, e)
  99  	}
 100  	return elems
 101  }
 102  
 103  // digestGoSource decomposes Go source code into structural elements.
 104  func digestGoSource(body []byte) []axiom.Element {
 105  	var elems []axiom.Element
 106  	lines := strings.Split(string(body), "\n")
 107  
 108  	for _, line := range lines {
 109  		trimmed := strings.TrimSpace(line)
 110  		if trimmed == "" || strings.HasPrefix(trimmed, "//") {
 111  			continue
 112  		}
 113  
 114  		// Classify Go source lines by structure.
 115  		switch {
 116  		case strings.HasPrefix(trimmed, "func "):
 117  			elems = append(elems, enzyme.HexElem("func", trimmed))
 118  		case strings.HasPrefix(trimmed, "type "):
 119  			elems = append(elems, enzyme.HexElem("type", trimmed))
 120  		case strings.HasPrefix(trimmed, "import"):
 121  			elems = append(elems, enzyme.HexElem("import", trimmed))
 122  		case strings.HasPrefix(trimmed, "package "):
 123  			elems = append(elems, enzyme.HexElem("package", trimmed))
 124  		default:
 125  			elems = append(elems, enzyme.HexElem("code", trimmed))
 126  		}
 127  	}
 128  
 129  	return elems
 130  }
 131  
 132  // extractWords splits text into word elements, skipping short words.
 133  func extractWords(text string) []axiom.Element {
 134  	var elems []axiom.Element
 135  	for _, w := range strings.Fields(text) {
 136  		clean := strings.Trim(w, ".,;:!?()[]{}\"'`*#-_")
 137  		if len(clean) > 2 {
 138  			elems = append(elems, enzyme.HexElem("word", clean))
 139  		}
 140  	}
 141  	return elems
 142  }
 143  
 144  // stripHTMLTags performs basic HTML tag removal.
 145  func stripHTMLTags(html string) string {
 146  	var b strings.Builder
 147  	inTag := false
 148  	for _, r := range html {
 149  		if r == '<' {
 150  			inTag = true
 151  			continue
 152  		}
 153  		if r == '>' {
 154  			inTag = false
 155  			b.WriteRune(' ')
 156  			continue
 157  		}
 158  		if !inTag {
 159  			b.WriteRune(r)
 160  		}
 161  	}
 162  	return b.String()
 163  }
 164