extract.go raw
1 package cartography
2
3 import (
4 "bytes"
5 "fmt"
6 "go/ast"
7 "go/format"
8 "go/parser"
9 "go/token"
10 "os"
11 "path/filepath"
12 "strings"
13 "unicode"
14 )
15
16 // skipDirs are directories skipped during extraction.
17 var skipDirs = map[string]bool{
18 "_output": true, ".git": true, "vendor": true, "node_modules": true,
19 "dist": true, "build": true, ".svelte-kit": true, ".next": true,
20 "coverage": true, "testdata": true,
21 }
22
23 // ExtractPackage extracts atlas entries from a single Go package directory.
24 // Used to ingest external packages (e.g., go/ast) into the atlas.
25 // pkgDir is the absolute path to the package source directory.
26 // importPath is the Go import path (e.g., "go/token").
27 func ExtractPackage(pkgDir string, importPath string) (*Atlas, error) {
28 atlas := NewAtlas()
29
30 entries, err := os.ReadDir(pkgDir)
31 if err != nil {
32 return nil, fmt.Errorf("read dir %s: %w", pkgDir, err)
33 }
34
35 for _, de := range entries {
36 if de.IsDir() {
37 continue
38 }
39 name := de.Name()
40 if filepath.Ext(name) != ".go" || strings.HasSuffix(name, "_test.go") {
41 continue
42 }
43 absPath := filepath.Join(pkgDir, name)
44 relPath := importPath + "/" + name
45 if err := extractFile(atlas, absPath, relPath); err != nil {
46 continue // skip unparseable
47 }
48 }
49
50 atlas.updateStats()
51 atlas.BuildIndexes()
52 return atlas, nil
53 }
54
55 // ExtractAll walks the project root and produces atlas entries for every
56 // Go declaration. Zero oracle calls. Uses go/ast for signatures, doc
57 // comments, parameter types, return types, and exported status.
58 func ExtractAll(projectRoot string) (*Atlas, error) {
59 atlas := NewAtlas()
60
61 err := filepath.Walk(projectRoot, func(path string, info os.FileInfo, err error) error {
62 if err != nil {
63 return nil
64 }
65 if info.IsDir() {
66 if skipDirs[info.Name()] {
67 return filepath.SkipDir
68 }
69 return nil
70 }
71 // Only .go files, skip tests.
72 if filepath.Ext(path) != ".go" {
73 return nil
74 }
75 if strings.HasSuffix(path, "_test.go") {
76 return nil
77 }
78
79 rel, _ := filepath.Rel(projectRoot, path)
80 if err := extractFile(atlas, path, rel); err != nil {
81 // Skip unparseable files silently.
82 return nil
83 }
84 return nil
85 })
86 if err != nil {
87 return nil, fmt.Errorf("walk: %w", err)
88 }
89
90 atlas.updateStats()
91 atlas.BuildIndexes()
92 return atlas, nil
93 }
94
95 // extractFile parses a single Go file and adds entries to the atlas.
96 func extractFile(atlas *Atlas, absPath, relPath string) error {
97 fset := token.NewFileSet()
98 file, err := parser.ParseFile(fset, absPath, nil, parser.ParseComments)
99 if err != nil {
100 return err
101 }
102
103 pkgName := ""
104 if file.Name != nil {
105 pkgName = file.Name.Name
106 }
107
108 for _, decl := range file.Decls {
109 switch d := decl.(type) {
110 case *ast.FuncDecl:
111 e := extractFunc(fset, file, d, pkgName, relPath)
112 if e != nil {
113 atlas.Entries[e.ID] = e
114 }
115 case *ast.GenDecl:
116 for _, spec := range d.Specs {
117 if ts, ok := spec.(*ast.TypeSpec); ok {
118 e := extractType(fset, file, d, ts, pkgName, relPath)
119 if e != nil {
120 atlas.Entries[e.ID] = e
121 }
122 }
123 }
124 }
125 }
126
127 return nil
128 }
129
130 // extractFunc creates an Entry from a function or method declaration.
131 func extractFunc(fset *token.FileSet, file *ast.File, fn *ast.FuncDecl, pkg, relPath string) *Entry {
132 name := fn.Name.Name
133 receiver := ""
134 kind := "func"
135 id := pkg + "." + name
136
137 if fn.Recv != nil && len(fn.Recv.List) > 0 {
138 kind = "method"
139 receiver = exprString(fset, fn.Recv.List[0].Type)
140 // Strip pointer.
141 recv := strings.TrimPrefix(receiver, "*")
142 id = pkg + "." + recv + "." + name
143 }
144
145 // Parameters.
146 params := extractParams(fset, fn.Type.Params)
147
148 // Returns.
149 returns := extractReturns(fset, fn.Type.Results)
150
151 // Signature.
152 sig := renderSignature(fset, fn, kind, receiver)
153
154 // Doc comment.
155 doc := ""
156 if fn.Doc != nil {
157 doc = strings.TrimSpace(fn.Doc.Text())
158 }
159
160 // Side effects (mechanical detection).
161 sideEffects := detectSideEffects(fn.Body)
162
163 // Mechanical description.
164 desc := buildFuncDescription(name, kind, receiver, params, returns, doc)
165
166 // Mechanical contract.
167 contract := buildContract(params, returns)
168
169 // Concepts.
170 concepts := extractConcepts(name, pkg, params, returns, doc)
171
172 line := fset.Position(fn.Pos()).Line
173
174 return &Entry{
175 ID: id,
176 Kind: kind,
177 Package: pkg,
178 Name: name,
179 Receiver: receiver,
180 Exported: ast.IsExported(name),
181 FilePath: relPath,
182 Line: line,
183 Signature: sig,
184 Params: params,
185 Returns: returns,
186 DocComment: doc,
187 SideEffects: sideEffects,
188 Description: desc,
189 Contract: contract,
190 Concepts: concepts,
191 Confidence: 0.3,
192 Source: "mechanical",
193 }
194 }
195
196 // extractType creates an Entry from a type declaration.
197 func extractType(fset *token.FileSet, file *ast.File, gd *ast.GenDecl, ts *ast.TypeSpec, pkg, relPath string) *Entry {
198 name := ts.Name.Name
199 id := pkg + "." + name
200 kind := "type"
201
202 // Doc comment — prefer type-level, fall back to GenDecl-level.
203 doc := ""
204 if ts.Doc != nil {
205 doc = strings.TrimSpace(ts.Doc.Text())
206 } else if gd.Doc != nil {
207 doc = strings.TrimSpace(gd.Doc.Text())
208 }
209
210 // Determine struct vs interface.
211 var desc string
212 var fields []string
213
214 switch t := ts.Type.(type) {
215 case *ast.StructType:
216 kind = "type"
217 fields = extractFieldNames(fset, t.Fields)
218 if len(fields) > 0 {
219 desc = fmt.Sprintf("Type %s is a struct with fields: %s.", name, strings.Join(fields, ", "))
220 } else {
221 desc = fmt.Sprintf("Type %s is a struct.", name)
222 }
223 case *ast.InterfaceType:
224 kind = "interface"
225 methods := extractInterfaceMethods(fset, t)
226 if len(methods) > 0 {
227 desc = fmt.Sprintf("Type %s is an interface with methods: %s.", name, strings.Join(methods, ", "))
228 } else {
229 desc = fmt.Sprintf("Type %s is an interface.", name)
230 }
231 default:
232 // Type alias or named type.
233 underlying := exprString(fset, ts.Type)
234 desc = fmt.Sprintf("Type %s is defined as %s.", name, underlying)
235 }
236
237 if doc != "" {
238 desc += " " + firstSentence(doc)
239 }
240
241 // Concepts.
242 concepts := splitCamelCase(name)
243 concepts = append(concepts, strings.ToLower(pkg))
244 for _, f := range fields {
245 concepts = append(concepts, strings.ToLower(f))
246 }
247
248 line := fset.Position(ts.Pos()).Line
249
250 return &Entry{
251 ID: id,
252 Kind: kind,
253 Package: pkg,
254 Name: name,
255 Exported: ast.IsExported(name),
256 FilePath: relPath,
257 Line: line,
258 Signature: fmt.Sprintf("type %s", name),
259 DocComment: doc,
260 Description: desc,
261 Concepts: dedupLower(concepts),
262 Confidence: 0.3,
263 Source: "mechanical",
264 }
265 }
266
267 // extractParams converts an ast.FieldList into ParamInfo slices.
268 func extractParams(fset *token.FileSet, fields *ast.FieldList) []ParamInfo {
269 if fields == nil {
270 return nil
271 }
272 var params []ParamInfo
273 for _, f := range fields.List {
274 typStr := exprString(fset, f.Type)
275 if len(f.Names) == 0 {
276 // Unnamed parameter.
277 params = append(params, ParamInfo{Type: typStr})
278 }
279 for _, n := range f.Names {
280 params = append(params, ParamInfo{Name: n.Name, Type: typStr})
281 }
282 }
283 return params
284 }
285
286 // extractReturns converts an ast.FieldList into ReturnInfo slices.
287 func extractReturns(fset *token.FileSet, fields *ast.FieldList) []ReturnInfo {
288 if fields == nil {
289 return nil
290 }
291 var returns []ReturnInfo
292 for _, f := range fields.List {
293 typStr := exprString(fset, f.Type)
294 ri := ReturnInfo{
295 Type: typStr,
296 IsError: typStr == "error",
297 }
298 returns = append(returns, ri)
299 }
300 return returns
301 }
302
303 // renderSignature produces a readable signature string.
304 func renderSignature(fset *token.FileSet, fn *ast.FuncDecl, kind, receiver string) string {
305 var b strings.Builder
306 if kind == "method" {
307 fmt.Fprintf(&b, "func (%s) %s", receiver, fn.Name.Name)
308 } else {
309 fmt.Fprintf(&b, "func %s", fn.Name.Name)
310 }
311 // Parameters.
312 b.WriteString("(")
313 if fn.Type.Params != nil {
314 for i, f := range fn.Type.Params.List {
315 if i > 0 {
316 b.WriteString(", ")
317 }
318 typStr := exprString(fset, f.Type)
319 if len(f.Names) > 0 {
320 names := make([]string, len(f.Names))
321 for j, n := range f.Names {
322 names[j] = n.Name
323 }
324 fmt.Fprintf(&b, "%s %s", strings.Join(names, ", "), typStr)
325 } else {
326 b.WriteString(typStr)
327 }
328 }
329 }
330 b.WriteString(")")
331
332 // Returns.
333 if fn.Type.Results != nil && len(fn.Type.Results.List) > 0 {
334 if len(fn.Type.Results.List) == 1 && len(fn.Type.Results.List[0].Names) == 0 {
335 b.WriteString(" ")
336 b.WriteString(exprString(fset, fn.Type.Results.List[0].Type))
337 } else {
338 b.WriteString(" (")
339 for i, f := range fn.Type.Results.List {
340 if i > 0 {
341 b.WriteString(", ")
342 }
343 b.WriteString(exprString(fset, f.Type))
344 }
345 b.WriteString(")")
346 }
347 }
348
349 return b.String()
350 }
351
352 // detectSideEffects scans a function body for common side-effect patterns.
353 func detectSideEffects(body *ast.BlockStmt) []string {
354 if body == nil {
355 return nil
356 }
357 var effects []string
358 seen := make(map[string]bool)
359
360 ast.Inspect(body, func(n ast.Node) bool {
361 switch node := n.(type) {
362 case *ast.GoStmt:
363 if !seen["spawns goroutine"] {
364 effects = append(effects, "spawns goroutine")
365 seen["spawns goroutine"] = true
366 }
367 case *ast.SendStmt:
368 if !seen["channel send"] {
369 effects = append(effects, "channel send")
370 seen["channel send"] = true
371 }
372 case *ast.CallExpr:
373 name := callName(node)
374 if strings.HasPrefix(name, "os.") && !seen["file I/O"] {
375 effects = append(effects, "file I/O")
376 seen["file I/O"] = true
377 }
378 if strings.HasPrefix(name, "http.") && !seen["network"] {
379 effects = append(effects, "network")
380 seen["network"] = true
381 }
382 if strings.HasPrefix(name, "exec.") && !seen["exec"] {
383 effects = append(effects, "exec")
384 seen["exec"] = true
385 }
386 }
387 return true
388 })
389 return effects
390 }
391
392 // callName extracts a readable name from a call expression.
393 func callName(call *ast.CallExpr) string {
394 switch fn := call.Fun.(type) {
395 case *ast.SelectorExpr:
396 if ident, ok := fn.X.(*ast.Ident); ok {
397 return ident.Name + "." + fn.Sel.Name
398 }
399 case *ast.Ident:
400 return fn.Name
401 }
402 return ""
403 }
404
405 // buildFuncDescription generates a mechanical English description of a function.
406 func buildFuncDescription(name, kind, receiver string, params []ParamInfo, returns []ReturnInfo, doc string) string {
407 var b strings.Builder
408
409 if kind == "method" {
410 fmt.Fprintf(&b, "Method %s on %s", name, receiver)
411 } else {
412 fmt.Fprintf(&b, "Function %s", name)
413 }
414
415 if len(params) > 0 {
416 paramStrs := make([]string, len(params))
417 for i, p := range params {
418 if p.Name != "" {
419 paramStrs[i] = p.Name + " " + p.Type
420 } else {
421 paramStrs[i] = p.Type
422 }
423 }
424 fmt.Fprintf(&b, " takes (%s)", strings.Join(paramStrs, ", "))
425 }
426
427 if len(returns) > 0 {
428 retStrs := make([]string, len(returns))
429 for i, r := range returns {
430 retStrs[i] = r.Type
431 }
432 fmt.Fprintf(&b, " returns (%s)", strings.Join(retStrs, ", "))
433 }
434
435 b.WriteString(".")
436
437 if doc != "" {
438 b.WriteString(" ")
439 b.WriteString(firstSentence(doc))
440 }
441
442 return b.String()
443 }
444
445 // buildContract generates a mechanical contract string.
446 func buildContract(params []ParamInfo, returns []ReturnInfo) string {
447 var parts []string
448
449 if len(params) > 0 {
450 types := make([]string, len(params))
451 for i, p := range params {
452 types[i] = p.Type
453 }
454 parts = append(parts, "Inputs: "+strings.Join(types, ", "))
455 }
456
457 if len(returns) > 0 {
458 types := make([]string, len(returns))
459 for i, r := range returns {
460 types[i] = r.Type
461 }
462 parts = append(parts, "Outputs: "+strings.Join(types, ", "))
463
464 for _, r := range returns {
465 if r.IsError {
466 parts = append(parts, "May return error")
467 break
468 }
469 }
470 }
471
472 if len(parts) == 0 {
473 return ""
474 }
475 return strings.Join(parts, ". ") + "."
476 }
477
478 // extractConcepts derives concept tags from a declaration.
479 func extractConcepts(name, pkg string, params []ParamInfo, returns []ReturnInfo, doc string) []string {
480 var concepts []string
481
482 // CamelCase split of the name.
483 concepts = append(concepts, splitCamelCase(name)...)
484
485 // Package name.
486 concepts = append(concepts, strings.ToLower(pkg))
487
488 // Parameter type base names.
489 for _, p := range params {
490 concepts = append(concepts, typeBaseName(p.Type))
491 }
492
493 // Return type base names.
494 for _, r := range returns {
495 if r.Type != "error" && r.Type != "bool" && r.Type != "string" &&
496 r.Type != "int" && r.Type != "float64" {
497 concepts = append(concepts, typeBaseName(r.Type))
498 }
499 }
500
501 // Significant doc comment words.
502 if doc != "" {
503 for _, w := range strings.Fields(doc) {
504 w = strings.ToLower(strings.Trim(w, ".,;:!?()"))
505 if len(w) >= 4 && !stopWords[w] {
506 concepts = append(concepts, w)
507 }
508 }
509 }
510
511 return dedupLower(concepts)
512 }
513
514 // splitCamelCase splits "ExtractSelfKnowledge" into ["extract", "self", "knowledge"].
515 func splitCamelCase(s string) []string {
516 var words []string
517 var current []rune
518
519 for _, r := range s {
520 if unicode.IsUpper(r) && len(current) > 0 {
521 words = append(words, strings.ToLower(string(current)))
522 current = current[:0]
523 }
524 current = append(current, r)
525 }
526 if len(current) > 0 {
527 words = append(words, strings.ToLower(string(current)))
528 }
529
530 // Filter out single-char words.
531 var result []string
532 for _, w := range words {
533 if len(w) >= 2 {
534 result = append(result, w)
535 }
536 }
537 return result
538 }
539
540 // typeBaseName extracts a base name from a Go type expression.
541 // "*lattice.Lattice" → "lattice", "[]int" → "int", "map[string]bool" → "".
542 func typeBaseName(t string) string {
543 t = strings.TrimPrefix(t, "*")
544 t = strings.TrimPrefix(t, "[]")
545 if i := strings.LastIndexByte(t, '.'); i >= 0 {
546 return strings.ToLower(t[i+1:])
547 }
548 return strings.ToLower(t)
549 }
550
551 // dedupLower deduplicates and lowercases concept tags.
552 func dedupLower(tags []string) []string {
553 seen := make(map[string]bool)
554 var result []string
555 for _, t := range tags {
556 t = strings.ToLower(t)
557 if t != "" && !seen[t] {
558 seen[t] = true
559 result = append(result, t)
560 }
561 }
562 return result
563 }
564
565 // exprString renders an ast.Expr back to Go source text.
566 func exprString(fset *token.FileSet, expr ast.Expr) string {
567 if expr == nil {
568 return ""
569 }
570 var buf bytes.Buffer
571 if err := format.Node(&buf, fset, expr); err != nil {
572 return fmt.Sprintf("%T", expr)
573 }
574 return buf.String()
575 }
576
577 // extractFieldNames returns the names of struct fields.
578 func extractFieldNames(fset *token.FileSet, fields *ast.FieldList) []string {
579 if fields == nil {
580 return nil
581 }
582 var names []string
583 for _, f := range fields.List {
584 for _, n := range f.Names {
585 names = append(names, n.Name)
586 }
587 }
588 return names
589 }
590
591 // extractInterfaceMethods returns the method signatures of an interface.
592 func extractInterfaceMethods(fset *token.FileSet, iface *ast.InterfaceType) []string {
593 if iface == nil || iface.Methods == nil {
594 return nil
595 }
596 var methods []string
597 for _, m := range iface.Methods.List {
598 for _, n := range m.Names {
599 methods = append(methods, n.Name)
600 }
601 }
602 return methods
603 }
604