package cartography import ( "bytes" "fmt" "go/ast" "go/format" "go/parser" "go/token" "os" "path/filepath" "strings" "unicode" ) // skipDirs are directories skipped during extraction. var skipDirs = map[string]bool{ "_output": true, ".git": true, "vendor": true, "node_modules": true, "dist": true, "build": true, ".svelte-kit": true, ".next": true, "coverage": true, "testdata": true, } // ExtractPackage extracts atlas entries from a single Go package directory. // Used to ingest external packages (e.g., go/ast) into the atlas. // pkgDir is the absolute path to the package source directory. // importPath is the Go import path (e.g., "go/token"). func ExtractPackage(pkgDir string, importPath string) (*Atlas, error) { atlas := NewAtlas() entries, err := os.ReadDir(pkgDir) if err != nil { return nil, fmt.Errorf("read dir %s: %w", pkgDir, err) } for _, de := range entries { if de.IsDir() { continue } name := de.Name() if filepath.Ext(name) != ".go" || strings.HasSuffix(name, "_test.go") { continue } absPath := filepath.Join(pkgDir, name) relPath := importPath + "/" + name if err := extractFile(atlas, absPath, relPath); err != nil { continue // skip unparseable } } atlas.updateStats() atlas.BuildIndexes() return atlas, nil } // ExtractAll walks the project root and produces atlas entries for every // Go declaration. Zero oracle calls. Uses go/ast for signatures, doc // comments, parameter types, return types, and exported status. func ExtractAll(projectRoot string) (*Atlas, error) { atlas := NewAtlas() err := filepath.Walk(projectRoot, func(path string, info os.FileInfo, err error) error { if err != nil { return nil } if info.IsDir() { if skipDirs[info.Name()] { return filepath.SkipDir } return nil } // Only .go files, skip tests. if filepath.Ext(path) != ".go" { return nil } if strings.HasSuffix(path, "_test.go") { return nil } rel, _ := filepath.Rel(projectRoot, path) if err := extractFile(atlas, path, rel); err != nil { // Skip unparseable files silently. return nil } return nil }) if err != nil { return nil, fmt.Errorf("walk: %w", err) } atlas.updateStats() atlas.BuildIndexes() return atlas, nil } // extractFile parses a single Go file and adds entries to the atlas. func extractFile(atlas *Atlas, absPath, relPath string) error { fset := token.NewFileSet() file, err := parser.ParseFile(fset, absPath, nil, parser.ParseComments) if err != nil { return err } pkgName := "" if file.Name != nil { pkgName = file.Name.Name } for _, decl := range file.Decls { switch d := decl.(type) { case *ast.FuncDecl: e := extractFunc(fset, file, d, pkgName, relPath) if e != nil { atlas.Entries[e.ID] = e } case *ast.GenDecl: for _, spec := range d.Specs { if ts, ok := spec.(*ast.TypeSpec); ok { e := extractType(fset, file, d, ts, pkgName, relPath) if e != nil { atlas.Entries[e.ID] = e } } } } } return nil } // extractFunc creates an Entry from a function or method declaration. func extractFunc(fset *token.FileSet, file *ast.File, fn *ast.FuncDecl, pkg, relPath string) *Entry { name := fn.Name.Name receiver := "" kind := "func" id := pkg + "." + name if fn.Recv != nil && len(fn.Recv.List) > 0 { kind = "method" receiver = exprString(fset, fn.Recv.List[0].Type) // Strip pointer. recv := strings.TrimPrefix(receiver, "*") id = pkg + "." + recv + "." + name } // Parameters. params := extractParams(fset, fn.Type.Params) // Returns. returns := extractReturns(fset, fn.Type.Results) // Signature. sig := renderSignature(fset, fn, kind, receiver) // Doc comment. doc := "" if fn.Doc != nil { doc = strings.TrimSpace(fn.Doc.Text()) } // Side effects (mechanical detection). sideEffects := detectSideEffects(fn.Body) // Mechanical description. desc := buildFuncDescription(name, kind, receiver, params, returns, doc) // Mechanical contract. contract := buildContract(params, returns) // Concepts. concepts := extractConcepts(name, pkg, params, returns, doc) line := fset.Position(fn.Pos()).Line return &Entry{ ID: id, Kind: kind, Package: pkg, Name: name, Receiver: receiver, Exported: ast.IsExported(name), FilePath: relPath, Line: line, Signature: sig, Params: params, Returns: returns, DocComment: doc, SideEffects: sideEffects, Description: desc, Contract: contract, Concepts: concepts, Confidence: 0.3, Source: "mechanical", } } // extractType creates an Entry from a type declaration. func extractType(fset *token.FileSet, file *ast.File, gd *ast.GenDecl, ts *ast.TypeSpec, pkg, relPath string) *Entry { name := ts.Name.Name id := pkg + "." + name kind := "type" // Doc comment — prefer type-level, fall back to GenDecl-level. doc := "" if ts.Doc != nil { doc = strings.TrimSpace(ts.Doc.Text()) } else if gd.Doc != nil { doc = strings.TrimSpace(gd.Doc.Text()) } // Determine struct vs interface. var desc string var fields []string switch t := ts.Type.(type) { case *ast.StructType: kind = "type" fields = extractFieldNames(fset, t.Fields) if len(fields) > 0 { desc = fmt.Sprintf("Type %s is a struct with fields: %s.", name, strings.Join(fields, ", ")) } else { desc = fmt.Sprintf("Type %s is a struct.", name) } case *ast.InterfaceType: kind = "interface" methods := extractInterfaceMethods(fset, t) if len(methods) > 0 { desc = fmt.Sprintf("Type %s is an interface with methods: %s.", name, strings.Join(methods, ", ")) } else { desc = fmt.Sprintf("Type %s is an interface.", name) } default: // Type alias or named type. underlying := exprString(fset, ts.Type) desc = fmt.Sprintf("Type %s is defined as %s.", name, underlying) } if doc != "" { desc += " " + firstSentence(doc) } // Concepts. concepts := splitCamelCase(name) concepts = append(concepts, strings.ToLower(pkg)) for _, f := range fields { concepts = append(concepts, strings.ToLower(f)) } line := fset.Position(ts.Pos()).Line return &Entry{ ID: id, Kind: kind, Package: pkg, Name: name, Exported: ast.IsExported(name), FilePath: relPath, Line: line, Signature: fmt.Sprintf("type %s", name), DocComment: doc, Description: desc, Concepts: dedupLower(concepts), Confidence: 0.3, Source: "mechanical", } } // extractParams converts an ast.FieldList into ParamInfo slices. func extractParams(fset *token.FileSet, fields *ast.FieldList) []ParamInfo { if fields == nil { return nil } var params []ParamInfo for _, f := range fields.List { typStr := exprString(fset, f.Type) if len(f.Names) == 0 { // Unnamed parameter. params = append(params, ParamInfo{Type: typStr}) } for _, n := range f.Names { params = append(params, ParamInfo{Name: n.Name, Type: typStr}) } } return params } // extractReturns converts an ast.FieldList into ReturnInfo slices. func extractReturns(fset *token.FileSet, fields *ast.FieldList) []ReturnInfo { if fields == nil { return nil } var returns []ReturnInfo for _, f := range fields.List { typStr := exprString(fset, f.Type) ri := ReturnInfo{ Type: typStr, IsError: typStr == "error", } returns = append(returns, ri) } return returns } // renderSignature produces a readable signature string. func renderSignature(fset *token.FileSet, fn *ast.FuncDecl, kind, receiver string) string { var b strings.Builder if kind == "method" { fmt.Fprintf(&b, "func (%s) %s", receiver, fn.Name.Name) } else { fmt.Fprintf(&b, "func %s", fn.Name.Name) } // Parameters. b.WriteString("(") if fn.Type.Params != nil { for i, f := range fn.Type.Params.List { if i > 0 { b.WriteString(", ") } typStr := exprString(fset, f.Type) if len(f.Names) > 0 { names := make([]string, len(f.Names)) for j, n := range f.Names { names[j] = n.Name } fmt.Fprintf(&b, "%s %s", strings.Join(names, ", "), typStr) } else { b.WriteString(typStr) } } } b.WriteString(")") // Returns. if fn.Type.Results != nil && len(fn.Type.Results.List) > 0 { if len(fn.Type.Results.List) == 1 && len(fn.Type.Results.List[0].Names) == 0 { b.WriteString(" ") b.WriteString(exprString(fset, fn.Type.Results.List[0].Type)) } else { b.WriteString(" (") for i, f := range fn.Type.Results.List { if i > 0 { b.WriteString(", ") } b.WriteString(exprString(fset, f.Type)) } b.WriteString(")") } } return b.String() } // detectSideEffects scans a function body for common side-effect patterns. func detectSideEffects(body *ast.BlockStmt) []string { if body == nil { return nil } var effects []string seen := make(map[string]bool) ast.Inspect(body, func(n ast.Node) bool { switch node := n.(type) { case *ast.GoStmt: if !seen["spawns goroutine"] { effects = append(effects, "spawns goroutine") seen["spawns goroutine"] = true } case *ast.SendStmt: if !seen["channel send"] { effects = append(effects, "channel send") seen["channel send"] = true } case *ast.CallExpr: name := callName(node) if strings.HasPrefix(name, "os.") && !seen["file I/O"] { effects = append(effects, "file I/O") seen["file I/O"] = true } if strings.HasPrefix(name, "http.") && !seen["network"] { effects = append(effects, "network") seen["network"] = true } if strings.HasPrefix(name, "exec.") && !seen["exec"] { effects = append(effects, "exec") seen["exec"] = true } } return true }) return effects } // callName extracts a readable name from a call expression. func callName(call *ast.CallExpr) string { switch fn := call.Fun.(type) { case *ast.SelectorExpr: if ident, ok := fn.X.(*ast.Ident); ok { return ident.Name + "." + fn.Sel.Name } case *ast.Ident: return fn.Name } return "" } // buildFuncDescription generates a mechanical English description of a function. func buildFuncDescription(name, kind, receiver string, params []ParamInfo, returns []ReturnInfo, doc string) string { var b strings.Builder if kind == "method" { fmt.Fprintf(&b, "Method %s on %s", name, receiver) } else { fmt.Fprintf(&b, "Function %s", name) } if len(params) > 0 { paramStrs := make([]string, len(params)) for i, p := range params { if p.Name != "" { paramStrs[i] = p.Name + " " + p.Type } else { paramStrs[i] = p.Type } } fmt.Fprintf(&b, " takes (%s)", strings.Join(paramStrs, ", ")) } if len(returns) > 0 { retStrs := make([]string, len(returns)) for i, r := range returns { retStrs[i] = r.Type } fmt.Fprintf(&b, " returns (%s)", strings.Join(retStrs, ", ")) } b.WriteString(".") if doc != "" { b.WriteString(" ") b.WriteString(firstSentence(doc)) } return b.String() } // buildContract generates a mechanical contract string. func buildContract(params []ParamInfo, returns []ReturnInfo) string { var parts []string if len(params) > 0 { types := make([]string, len(params)) for i, p := range params { types[i] = p.Type } parts = append(parts, "Inputs: "+strings.Join(types, ", ")) } if len(returns) > 0 { types := make([]string, len(returns)) for i, r := range returns { types[i] = r.Type } parts = append(parts, "Outputs: "+strings.Join(types, ", ")) for _, r := range returns { if r.IsError { parts = append(parts, "May return error") break } } } if len(parts) == 0 { return "" } return strings.Join(parts, ". ") + "." } // extractConcepts derives concept tags from a declaration. func extractConcepts(name, pkg string, params []ParamInfo, returns []ReturnInfo, doc string) []string { var concepts []string // CamelCase split of the name. concepts = append(concepts, splitCamelCase(name)...) // Package name. concepts = append(concepts, strings.ToLower(pkg)) // Parameter type base names. for _, p := range params { concepts = append(concepts, typeBaseName(p.Type)) } // Return type base names. for _, r := range returns { if r.Type != "error" && r.Type != "bool" && r.Type != "string" && r.Type != "int" && r.Type != "float64" { concepts = append(concepts, typeBaseName(r.Type)) } } // Significant doc comment words. if doc != "" { for _, w := range strings.Fields(doc) { w = strings.ToLower(strings.Trim(w, ".,;:!?()")) if len(w) >= 4 && !stopWords[w] { concepts = append(concepts, w) } } } return dedupLower(concepts) } // splitCamelCase splits "ExtractSelfKnowledge" into ["extract", "self", "knowledge"]. func splitCamelCase(s string) []string { var words []string var current []rune for _, r := range s { if unicode.IsUpper(r) && len(current) > 0 { words = append(words, strings.ToLower(string(current))) current = current[:0] } current = append(current, r) } if len(current) > 0 { words = append(words, strings.ToLower(string(current))) } // Filter out single-char words. var result []string for _, w := range words { if len(w) >= 2 { result = append(result, w) } } return result } // typeBaseName extracts a base name from a Go type expression. // "*lattice.Lattice" → "lattice", "[]int" → "int", "map[string]bool" → "". func typeBaseName(t string) string { t = strings.TrimPrefix(t, "*") t = strings.TrimPrefix(t, "[]") if i := strings.LastIndexByte(t, '.'); i >= 0 { return strings.ToLower(t[i+1:]) } return strings.ToLower(t) } // dedupLower deduplicates and lowercases concept tags. func dedupLower(tags []string) []string { seen := make(map[string]bool) var result []string for _, t := range tags { t = strings.ToLower(t) if t != "" && !seen[t] { seen[t] = true result = append(result, t) } } return result } // exprString renders an ast.Expr back to Go source text. func exprString(fset *token.FileSet, expr ast.Expr) string { if expr == nil { return "" } var buf bytes.Buffer if err := format.Node(&buf, fset, expr); err != nil { return fmt.Sprintf("%T", expr) } return buf.String() } // extractFieldNames returns the names of struct fields. func extractFieldNames(fset *token.FileSet, fields *ast.FieldList) []string { if fields == nil { return nil } var names []string for _, f := range fields.List { for _, n := range f.Names { names = append(names, n.Name) } } return names } // extractInterfaceMethods returns the method signatures of an interface. func extractInterfaceMethods(fset *token.FileSet, iface *ast.InterfaceType) []string { if iface == nil || iface.Methods == nil { return nil } var methods []string for _, m := range iface.Methods.List { for _, n := range m.Names { methods = append(methods, n.Name) } } return methods }