package forage import ( "strings" "git.mleku.dev/mleku/dendrite/pkg/axiom" "git.mleku.dev/mleku/dendrite/pkg/enzyme" ) // Digest routes fetched content through the appropriate enzyme based // on content type and URL. Returns typed lattice elements. func Digest(result *FetchResult) []axiom.Element { if result.StatusCode != 200 || len(result.Body) == 0 { return nil } ct := strings.ToLower(result.ContentType) switch { case strings.Contains(ct, "text/html"): return digestHTML(result.Body) case strings.Contains(ct, "text/markdown"), strings.HasSuffix(result.URL, ".md"): return digestMarkdown(result.Body) case strings.Contains(ct, "text/plain"), strings.Contains(ct, "application/json"): return digestText(result.Body) case strings.HasSuffix(result.URL, ".go"): return digestGoSource(result.Body) default: return digestText(result.Body) } } // digestMarkdown decomposes markdown into typed elements. // Headers become "heading" elements, code blocks become "code" elements, // prose becomes "word" elements (same as text enzyme). func digestMarkdown(body []byte) []axiom.Element { var elems []axiom.Element lines := strings.Split(string(body), "\n") inCodeBlock := false for _, line := range lines { trimmed := strings.TrimSpace(line) if strings.HasPrefix(trimmed, "```") { inCodeBlock = !inCodeBlock continue } if inCodeBlock { if trimmed != "" { elems = append(elems, enzyme.HexElem("code", trimmed)) } continue } if trimmed == "" { continue } // Headings. if strings.HasPrefix(trimmed, "#") { heading := strings.TrimLeft(trimmed, "# ") if heading != "" { elems = append(elems, enzyme.HexElem("heading", heading)) } continue } // Extract words from prose. elems = append(elems, extractWords(trimmed)...) } return elems } // digestHTML strips tags and decomposes the text content. func digestHTML(body []byte) []axiom.Element { // Simple HTML stripping — no dependency on a full parser. text := stripHTMLTags(string(body)) return digestText([]byte(text)) } // digestText decomposes plain text into word elements using the // text enzyme pattern. func digestText(body []byte) []axiom.Element { var elems []axiom.Element ch := enzyme.Text{}.Digest(strings.NewReader(string(body))) for e := range ch { if e.Type() == "space" { continue } elems = append(elems, e) } return elems } // digestGoSource decomposes Go source code into structural elements. func digestGoSource(body []byte) []axiom.Element { var elems []axiom.Element lines := strings.Split(string(body), "\n") for _, line := range lines { trimmed := strings.TrimSpace(line) if trimmed == "" || strings.HasPrefix(trimmed, "//") { continue } // Classify Go source lines by structure. switch { case strings.HasPrefix(trimmed, "func "): elems = append(elems, enzyme.HexElem("func", trimmed)) case strings.HasPrefix(trimmed, "type "): elems = append(elems, enzyme.HexElem("type", trimmed)) case strings.HasPrefix(trimmed, "import"): elems = append(elems, enzyme.HexElem("import", trimmed)) case strings.HasPrefix(trimmed, "package "): elems = append(elems, enzyme.HexElem("package", trimmed)) default: elems = append(elems, enzyme.HexElem("code", trimmed)) } } return elems } // extractWords splits text into word elements, skipping short words. func extractWords(text string) []axiom.Element { var elems []axiom.Element for _, w := range strings.Fields(text) { clean := strings.Trim(w, ".,;:!?()[]{}\"'`*#-_") if len(clean) > 2 { elems = append(elems, enzyme.HexElem("word", clean)) } } return elems } // stripHTMLTags performs basic HTML tag removal. func stripHTMLTags(html string) string { var b strings.Builder inTag := false for _, r := range html { if r == '<' { inTag = true continue } if r == '>' { inTag = false b.WriteRune(' ') continue } if !inTag { b.WriteRune(r) } } return b.String() }