mxlist.go raw
1 // mxlist.go implements package discovery for .mx source files,
2 // replacing the external `go list -json -deps` command.
3 //
4 // It uses go/build.Context with custom ReadDir/OpenFile hooks (from
5 // mxbuild.go) to find .mx files, then walks the dependency graph to
6 // produce the same PackageJSON structures the loader expects.
7
8 package loader
9
10 import (
11 "fmt"
12 "go/build"
13 "os"
14 "path/filepath"
15 "strings"
16
17 "moxie/compileopts"
18 "moxie/goenv"
19 )
20
21 // mxListPackages discovers packages and their transitive dependencies,
22 // returning them in dependency order (dependencies before dependents).
23 // This replaces `go list -json -deps -e`.
24 func mxListPackages(config *compileopts.Config, goroot string, inputPkg string) ([]*PackageJSON, error) {
25 return mxListPackagesImpl(config, goroot, inputPkg, false)
26 }
27
28 func mxListPackagesTest(config *compileopts.Config, goroot string, inputPkg string) ([]*PackageJSON, error) {
29 return mxListPackagesImpl(config, goroot, inputPkg, true)
30 }
31
32 func mxListPackagesImpl(config *compileopts.Config, goroot string, inputPkg string, testMode bool) ([]*PackageJSON, error) {
33 ctx := mxContext(goroot, config.GOOS(), config.GOARCH(), config.BuildTags())
34
35 // Determine the working directory and module info.
36 wd := config.Options.Directory
37 if wd == "" {
38 var err error
39 wd, err = os.Getwd()
40 if err != nil {
41 return nil, err
42 }
43 }
44
45 modPath, modDir, modGoVersion, err := readModInfo(wd)
46 if err != nil {
47 return nil, fmt.Errorf("mxlist: %w", err)
48 }
49
50 modCache := gomodcache()
51
52 // Parse module require and replace directives.
53 modFile := findModFile(modDir)
54 requires, err := readModRequires(modFile)
55 if err != nil {
56 return nil, fmt.Errorf("mxlist: reading module requires: %w", err)
57 }
58 replaces, err := readModReplaces(modFile, modDir)
59 if err != nil {
60 return nil, fmt.Errorf("mxlist: reading module replaces: %w", err)
61 }
62
63 r := &resolver{
64 ctx: ctx,
65 modPath: modPath,
66 modDir: modDir,
67 modGoVersion: modGoVersion,
68 modCache: modCache,
69 requires: requires,
70 replaces: replaces,
71 goroot: goroot,
72 mxhDir: MXHProtocolsDir(),
73 seen: make(map[string]*PackageJSON),
74 }
75
76 // The runtime package is an implicit dependency — never explicitly
77 // imported in user source, but always required by the compiler.
78 // Walk it BEFORE the user package so that MainPkg() (which returns
79 // the last element of sorted) correctly identifies the user package.
80 if err := r.walk("runtime", ""); err != nil {
81 return nil, err
82 }
83
84 // The moxie package is an implicit dependency — the compiler needs it
85 // to look up the Codec interface for spawn-boundary type checking.
86 // Like runtime, it lives in the synthetic GOROOT at src/moxie/.
87 if err := r.walk("moxie", ""); err != nil {
88 return nil, err
89 }
90
91 err = r.walk(inputPkg, wd)
92 if err != nil {
93 return nil, err
94 }
95
96 if testMode {
97 sorted, err := r.addTestMain(inputPkg, wd)
98 if err != nil {
99 return nil, err
100 }
101 return sorted, nil
102 }
103
104 return r.sorted, nil
105 }
106
107 // MxListImportPaths returns just the import paths of the given packages,
108 // using the native .mx-aware resolver instead of `go list`.
109 func MxListImportPaths(config *compileopts.Config, pkgs []string) ([]string, error) {
110 goroot, err := GetCachedGoroot(config)
111 if err != nil {
112 return nil, err
113 }
114 var result []string
115 for _, pkg := range pkgs {
116 pkgJSONs, err := mxListPackages(config, goroot, pkg)
117 if err != nil {
118 return nil, err
119 }
120 if len(pkgJSONs) > 0 {
121 last := pkgJSONs[len(pkgJSONs)-1]
122 result = append(result, last.ImportPath)
123 }
124 }
125 return result, nil
126 }
127
128 // addTestMain finds _test.mx files in the target package, merges them,
129 // scans for Test* functions, generates a synthetic test main, and appends
130 // a .test main package to the resolver's sorted list.
131 func (r *resolver) addTestMain(inputPkg string, wd string) ([]*PackageJSON, error) {
132 // Find the target package.
133 var targetPkg *PackageJSON
134 for _, pj := range r.sorted {
135 if pj.ImportPath == inputPkg || (inputPkg == "." && pj.Dir == wd) {
136 targetPkg = pj
137 break
138 }
139 }
140 if targetPkg == nil {
141 return r.sorted, nil
142 }
143
144 // Find _test.mx files in the package directory.
145 entries, err := os.ReadDir(targetPkg.Dir)
146 if err != nil {
147 return nil, err
148 }
149 var testFiles []string
150 for _, e := range entries {
151 name := e.Name()
152 if strings.HasSuffix(name, "_test.mx") && !e.IsDir() {
153 testFiles = append(testFiles, name)
154 }
155 }
156 if len(testFiles) == 0 {
157 return nil, NoTestFilesError{ImportPath: targetPkg.ImportPath}
158 }
159
160 // Merge test files into the target package.
161 targetPkg.GoFiles = append(targetPkg.GoFiles, testFiles...)
162 // Mark this package as the test package.
163 targetPkg.ForTest = targetPkg.ImportPath
164
165 // Remove target package from sorted — we need to re-add it after
166 // walking test dependencies so the dependency order is correct.
167 var targetIdx int
168 for i, pj := range r.sorted {
169 if pj == targetPkg {
170 targetIdx = i
171 break
172 }
173 }
174 copy(r.sorted[targetIdx:], r.sorted[targetIdx+1:])
175 r.sorted = r.sorted[:len(r.sorted)-1]
176 delete(r.seen, targetPkg.ImportPath)
177
178 // Walk testing dependency (test files import "testing").
179 if err := r.walk("testing", targetPkg.Dir); err != nil {
180 return nil, fmt.Errorf("walking testing dependency: %w", err)
181 }
182 // Add "testing" to the package's imports.
183 hasTestingImport := false
184 for _, imp := range targetPkg.Imports {
185 if imp == "testing" {
186 hasTestingImport = true
187 break
188 }
189 }
190 if !hasTestingImport {
191 targetPkg.Imports = append(targetPkg.Imports, "testing")
192 }
193
194 // Re-add target package after its test dependencies.
195 r.seen[targetPkg.ImportPath] = targetPkg
196 r.sorted = append(r.sorted, targetPkg)
197
198 // Scan test files for Test* function names.
199 var testFuncs []string
200 for _, tf := range testFiles {
201 data, err := os.ReadFile(filepath.Join(targetPkg.Dir, tf))
202 if err != nil {
203 return nil, err
204 }
205 for _, line := range strings.Split(string(data), "\n") {
206 line = strings.TrimSpace(line)
207 if strings.HasPrefix(line, "func Test") {
208 // Extract function name: "func TestFoo(t *testing.T) {"
209 name := line[len("func "):]
210 if idx := strings.IndexByte(name, '('); idx > 0 {
211 name = name[:idx]
212 testFuncs = append(testFuncs, name)
213 }
214 }
215 }
216 }
217
218 // Generate synthetic _testmain.mx in a temp directory.
219 tmpDir, err := os.MkdirTemp("", "moxie-testmain-*")
220 if err != nil {
221 return nil, err
222 }
223
224 var buf strings.Builder
225 buf.WriteString("package main\n\nimport (\n")
226 buf.WriteString("\t\"testing\"\n")
227 buf.WriteString("\tpkg \"" + targetPkg.ImportPath + "\"\n")
228 buf.WriteString(")\n\nfunc main() {\n")
229 buf.WriteString("\ttests := []testing.InternalTest{\n")
230 for _, fn := range testFuncs {
231 buf.WriteString("\t\t{\"" + fn + "\", pkg." + fn + "},\n")
232 }
233 buf.WriteString("\t}\n")
234 buf.WriteString("\ttesting.Main(tests)\n")
235 buf.WriteString("}\n")
236
237 mainFile := filepath.Join(tmpDir, "_testmain.mx")
238 if err := os.WriteFile(mainFile, []byte(buf.String()), 0644); err != nil {
239 return nil, err
240 }
241
242 // Add the .test main package.
243 testMainPkg := &PackageJSON{
244 Dir: tmpDir,
245 ImportPath: targetPkg.ImportPath + ".test",
246 Name: "main",
247 GoFiles: []string{"_testmain.mx"},
248 Imports: []string{"testing", targetPkg.ImportPath},
249 }
250
251 r.sorted = append(r.sorted, testMainPkg)
252 return r.sorted, nil
253 }
254
255 // resolver walks the dependency graph, building PackageJSON entries.
256 type resolver struct {
257 ctx build.Context
258 modPath string
259 modDir string
260 modGoVersion string
261 modCache string
262 requires map[string]string // module path → version
263 replaces map[string]string // module path → absolute local dir
264 goroot string
265 mxhDir string // path to $GOCACHE/mxinstall/protocols/ for .mxh fallback
266 seen map[string]*PackageJSON
267 sorted []*PackageJSON
268 }
269
270 // walk imports a package and recursively walks its dependencies.
271 func (r *resolver) walk(importPath string, srcDir string) error {
272 if r.seen[importPath] != nil {
273 return nil
274 }
275 if importPath == "C" {
276 // CGo pseudo-import, skip.
277 return nil
278 }
279 if importPath == "unsafe" {
280 // Built-in package, no files to discover.
281 pj := &PackageJSON{
282 ImportPath: "unsafe",
283 Name: "unsafe",
284 }
285 r.seen[importPath] = pj
286 r.sorted = append(r.sorted, pj)
287 return nil
288 }
289
290 // Mark as seen (with nil) to break import cycles.
291 r.seen[importPath] = &PackageJSON{}
292
293 dir, err := r.resolveDir(importPath, srcDir)
294 if err != nil {
295 // Check for a cached .mxh before failing.
296 if r.mxhDir != "" {
297 mxhPath := filepath.Join(r.mxhDir, importPath+".mxh")
298 if _, statErr := os.Stat(mxhPath); statErr == nil {
299 pj := &PackageJSON{
300 ImportPath: importPath,
301 Name: importPath[strings.LastIndex(importPath, "/")+1:],
302 IsMXH: true,
303 }
304 r.seen[importPath] = pj
305 r.sorted = append(r.sorted, pj)
306 return nil
307 }
308 }
309 return fmt.Errorf("resolving %q: %w\n\thint: run `moxie install %s` to cache this package", importPath, err, importPath)
310 }
311
312 pkg, err := r.ctx.ImportDir(dir, 0)
313 if err != nil {
314 // Check for NoGoError — might have only test files or only .mx files
315 // that our context didn't find (shouldn't happen with our hooks).
316 if _, ok := err.(*build.NoGoError); !ok {
317 return fmt.Errorf("importing %q from %s: %w", importPath, dir, err)
318 }
319 }
320
321 // Walk dependencies first (depth-first, deps before dependents).
322 allImports := pkg.Imports
323 for _, imp := range allImports {
324 if false {
325 fmt.Fprintf(os.Stderr, "DEBUG: %q imports %q (dir=%s)\n", importPath, imp, dir)
326 }
327 if err := r.walk(imp, dir); err != nil {
328 return err
329 }
330 }
331
332 // Build PackageJSON, mapping .go names back to .mx.
333 pj := &PackageJSON{
334 Dir: dir,
335 ImportPath: importPath,
336 Name: pkg.Name,
337 GoFiles: goToMx(pkg.GoFiles, dir),
338 CgoFiles: goToMx(pkg.CgoFiles, dir),
339 CFiles: pkg.CFiles,
340 Imports: pkg.Imports,
341 EmbedFiles: pkg.EmbedPatterns,
342 }
343
344 // Set module info for packages within our module.
345 if r.modPath != "" && (strings.HasPrefix(importPath, r.modPath+"/") || importPath == r.modPath) {
346 pj.Module.Path = r.modPath
347 pj.Module.Main = true
348 pj.Module.Dir = r.modDir
349 pj.Module.GoMod = findModFile(r.modDir)
350 pj.Module.GoVersion = r.modGoVersion
351 }
352
353 r.seen[importPath] = pj
354 r.sorted = append(r.sorted, pj)
355 return nil
356 }
357
358 // resolveDir maps an import path to an on-disk directory.
359 func (r *resolver) resolveDir(importPath string, srcDir string) (string, error) {
360 // 1. Local/relative imports.
361 if importPath == "." || strings.HasPrefix(importPath, "./") || strings.HasPrefix(importPath, "../") {
362 return filepath.Join(srcDir, importPath), nil
363 }
364
365 // 2. Standard library (in synthetic GOROOT).
366 stdDir := filepath.Join(r.goroot, "src", importPath)
367 if isDir(stdDir) {
368 return stdDir, nil
369 }
370 // 2b. Stdlib-vendored packages (e.g. golang.org/x/crypto).
371 stdVendorDir := filepath.Join(r.goroot, "src", "vendor", importPath)
372 if isDir(stdVendorDir) {
373 return stdVendorDir, nil
374 }
375
376 // 3. Module-local packages.
377 if r.modPath != "" && (strings.HasPrefix(importPath, r.modPath+"/") || importPath == r.modPath) {
378 rel := strings.TrimPrefix(importPath, r.modPath)
379 rel = strings.TrimPrefix(rel, "/")
380 dir := filepath.Join(r.modDir, rel)
381 if isDir(dir) {
382 return dir, nil
383 }
384 return "", fmt.Errorf("package %q not found in module at %s", importPath, dir)
385 }
386
387 // 4a. Local replace directives.
388 for mod, localDir := range r.replaces {
389 if importPath == mod || strings.HasPrefix(importPath, mod+"/") {
390 rel := strings.TrimPrefix(importPath, mod)
391 rel = strings.TrimPrefix(rel, "/")
392 dir := filepath.Join(localDir, rel)
393 if isDir(dir) {
394 return dir, nil
395 }
396 return "", fmt.Errorf("package %q not found in replace at %s", importPath, dir)
397 }
398 }
399
400 // 4b. Vendored packages (checked before module cache).
401 if r.modDir != "" {
402 vendorDir := filepath.Join(r.modDir, "vendor", importPath)
403 if isDir(vendorDir) {
404 return vendorDir, nil
405 }
406 }
407
408 // 4c. External dependencies (module cache — fallback when no vendor).
409 bestMod := ""
410 bestVer := ""
411 for mod, ver := range r.requires {
412 if (importPath == mod || strings.HasPrefix(importPath, mod+"/")) && len(mod) > len(bestMod) {
413 bestMod = mod
414 bestVer = ver
415 }
416 }
417 if bestMod != "" {
418 modCacheDir := filepath.Join(r.modCache, bestMod+"@"+bestVer)
419 rel := strings.TrimPrefix(importPath, bestMod)
420 rel = strings.TrimPrefix(rel, "/")
421 dir := filepath.Join(modCacheDir, rel)
422 if isDir(dir) {
423 return dir, nil
424 }
425 return "", fmt.Errorf("package %q not found in module cache at %s", importPath, dir)
426 }
427
428 return "", fmt.Errorf("package %q not found (not in stdlib, module, or cache)", importPath)
429 }
430
431 // findModFile returns the path of the module file in dir.
432 // Prefers moxie.mod, falls back to go.mod.
433 func findModFile(dir string) string {
434 mxmod := filepath.Join(dir, "moxie.mod")
435 if _, err := os.Stat(mxmod); err == nil {
436 return mxmod
437 }
438 return filepath.Join(dir, "go.mod")
439 }
440
441 // readModInfo reads the module path and directory from moxie.mod (or go.mod).
442 func readModInfo(startDir string) (modPath, modDir, goVersion string, err error) {
443 dir := startDir
444 for {
445 modFile := findModFile(dir)
446 data, err := os.ReadFile(modFile)
447 if err == nil {
448 modPath, goVersion = parseGoMod(string(data))
449 return modPath, dir, goVersion, nil
450 }
451 parent := filepath.Dir(dir)
452 if parent == dir {
453 return "", startDir, "", nil // no module file found, use wd
454 }
455 dir = parent
456 }
457 }
458
459 // parseGoMod extracts the module path, go version, and moxie version from
460 // go.mod or moxie.mod content.
461 func parseGoMod(content string) (modPath, goVersion string) {
462 for _, line := range strings.Split(content, "\n") {
463 line = strings.TrimSpace(line)
464 if strings.HasPrefix(line, "module ") {
465 modPath = strings.TrimSpace(strings.TrimPrefix(line, "module"))
466 }
467 if strings.HasPrefix(line, "go ") {
468 goVersion = strings.TrimSpace(strings.TrimPrefix(line, "go"))
469 }
470 if strings.HasPrefix(line, "moxie ") {
471 goVersion = strings.TrimSpace(strings.TrimPrefix(line, "moxie"))
472 }
473 }
474 return
475 }
476
477 // ParseMoxieModVersion reads a moxie.mod file and returns the moxie version
478 // string (e.g. "1.2.0"). Returns empty string if not found.
479 func ParseMoxieModVersion(dir string) string {
480 for _, name := range []string{"moxie.mod", "go.mod"} {
481 data, err := os.ReadFile(filepath.Join(dir, name))
482 if err != nil {
483 continue
484 }
485 for _, line := range strings.Split(string(data), "\n") {
486 line = strings.TrimSpace(line)
487 if strings.HasPrefix(line, "moxie ") {
488 return strings.TrimSpace(strings.TrimPrefix(line, "moxie"))
489 }
490 }
491 }
492 return ""
493 }
494
495 // readModRequires parses require directives from go.mod.
496 func readModRequires(gomodPath string) (map[string]string, error) {
497 data, err := os.ReadFile(gomodPath)
498 if err != nil {
499 if os.IsNotExist(err) {
500 return nil, nil
501 }
502 return nil, err
503 }
504
505 requires := make(map[string]string)
506 inRequire := false
507 for _, line := range strings.Split(string(data), "\n") {
508 line = strings.TrimSpace(line)
509
510 if line == ")" {
511 inRequire = false
512 continue
513 }
514 if strings.HasPrefix(line, "require (") {
515 inRequire = true
516 continue
517 }
518 if strings.HasPrefix(line, "require ") && !strings.Contains(line, "(") {
519 // Single-line require.
520 parts := strings.Fields(line)
521 if len(parts) >= 3 {
522 requires[parts[1]] = parts[2]
523 }
524 continue
525 }
526 if inRequire {
527 parts := strings.Fields(line)
528 if len(parts) >= 2 && !strings.HasPrefix(parts[0], "//") {
529 requires[parts[0]] = parts[1]
530 }
531 }
532 }
533 return requires, nil
534 }
535
536 // readModReplaces parses replace directives that point to local directories.
537 func readModReplaces(gomodPath, modDir string) (map[string]string, error) {
538 data, err := os.ReadFile(gomodPath)
539 if err != nil {
540 if os.IsNotExist(err) {
541 return nil, nil
542 }
543 return nil, err
544 }
545 replaces := make(map[string]string)
546 inReplace := false
547 for _, line := range strings.Split(string(data), "\n") {
548 line = strings.TrimSpace(line)
549 if line == ")" {
550 inReplace = false
551 continue
552 }
553 if strings.HasPrefix(line, "replace (") {
554 inReplace = true
555 continue
556 }
557 parseLine := ""
558 if strings.HasPrefix(line, "replace ") && !strings.Contains(line, "(") {
559 parseLine = strings.TrimPrefix(line, "replace ")
560 } else if inReplace && !strings.HasPrefix(line, "//") && line != "" {
561 parseLine = line
562 }
563 if parseLine == "" {
564 continue
565 }
566 parts := strings.Split(parseLine, "=>")
567 if len(parts) != 2 {
568 continue
569 }
570 lhs := strings.Fields(strings.TrimSpace(parts[0]))
571 rhs := strings.Fields(strings.TrimSpace(parts[1]))
572 if len(lhs) < 1 || len(rhs) < 1 {
573 continue
574 }
575 replacePath := rhs[0]
576 if strings.HasPrefix(replacePath, ".") || strings.HasPrefix(replacePath, "/") {
577 if !filepath.IsAbs(replacePath) {
578 replacePath = filepath.Join(modDir, replacePath)
579 }
580 replaces[lhs[0]] = filepath.Clean(replacePath)
581 }
582 }
583 return replaces, nil
584 }
585
586 func gomodcache() string {
587 c := goenv.Get("GOMODCACHE")
588 if c == "" {
589 gopath := goenv.Get("GOPATH")
590 if gopath == "" {
591 gopath = filepath.Join(os.Getenv("HOME"), "go")
592 }
593 c = filepath.Join(gopath, "pkg", "mod")
594 }
595 return c
596 }
597
598 func isDir(path string) bool {
599 fi, err := os.Stat(path)
600 return err == nil && fi.IsDir()
601 }
602