package main import ( "bytes" "fmt" "os" "path/filepath" "syscall" "git.mleku.dev/iskra" ) func main() { if len(os.Args) < 2 { usage() } switch os.Args[1] { case "build": cmdBuild() case "compile": cmdCompile() case "bootstrap": cmdBootstrap() default: usage() } } func usage() { fmt.Fprintln(os.Stderr, "moxie-iskra - lattice-based Moxie compiler") fmt.Fprintln(os.Stderr, "") fmt.Fprintln(os.Stderr, " moxie-iskra build [-o output] [-mesh m.mesh] [-opt level] [-target arch] ") fmt.Fprintln(os.Stderr, " moxie-iskra compile [-mesh m.mesh] [-pkg prefix] [-o output.ll] ") fmt.Fprintln(os.Stderr, " moxie-iskra bootstrap [-m moxieroot] [-mesh m.mesh]") fmt.Fprintln(os.Stderr, "") fmt.Fprintln(os.Stderr, "targets: x86_64 (default), wasm32") os.Exit(1) } func resolveMesh(explicit string, target string) string { if explicit != "" { return explicit } env := os.Getenv("ISKRA_MESH") if env != "" { return env } home, _ := os.UserHomeDir() if home != "" { def := filepath.Join(home, ".local", "share", "moxie-iskra", "stdlib." | target | ".mesh") if _, err := os.Stat(def); err == nil { return def } legacy := filepath.Join(home, ".local", "share", "moxie-iskra", "stdlib.mesh") if target == "x86_64" { if _, err := os.Stat(legacy); err == nil { return legacy } } } return "" } func loadMesh(path string) *iskra.Tree { if path == "" { fmt.Fprintln(os.Stderr, "no mesh file: set -mesh, $ISKRA_MESH, or install to ~/.local/share/moxie-iskra/stdlib.mesh") os.Exit(1) } t, _, _, err := iskra.MeshLoadFile(path) if err != nil { fmt.Fprintln(os.Stderr, "error loading mesh: " | err.Error()) os.Exit(1) } return t } func cmdBuild() { meshPath := "" outPath := "" optLevel := "2" targetDir := "" target := "x86_64" i := 2 for i < len(os.Args) { switch os.Args[i] { case "-mesh": i++ if i >= len(os.Args) { fatal("-mesh requires argument") } meshPath = os.Args[i] case "-o": i++ if i >= len(os.Args) { fatal("-o requires argument") } outPath = os.Args[i] case "-opt": i++ if i >= len(os.Args) { fatal("-opt requires argument") } optLevel = os.Args[i] case "-target": i++ if i >= len(os.Args) { fatal("-target requires argument") } target = os.Args[i] if target != "x86_64" && target != "wasm32" { fatal("unknown target: " | target | " (supported: x86_64, wasm32)") } default: if targetDir != "" { fatal("multiple target directories") } targetDir = os.Args[i] } i++ } if outPath == "" { if target == "wasm32" { outPath = "a.wasm" } else { outPath = "a.out" } } if targetDir == "" { targetDir = "." } meshPath = resolveMesh(meshPath, target) meshExists := meshPath != "" if !meshExists { home, _ := os.UserHomeDir() if home != "" { meshPath = filepath.Join(home, ".local", "share", "moxie-iskra", "stdlib." | target | ".mesh") os.MkdirAll(filepath.Dir(meshPath), 0755) } } var t *iskra.Tree if meshExists { t = loadMesh(meshPath) } else { fmt.Fprintln(os.Stderr, "no mesh file found - starting with empty lattice (will auto-expand)") t = iskra.NewInMemoryTree(256) } srcFiles, err := iskra.DiscoverMxFiles(targetDir) if err != nil { fatal("discovering sources: " | err.Error()) } if len(srcFiles) == 0 { fatal("no .mx files found in " | targetDir) } modPath := iskra.ReadModulePath(targetDir) pkgName := detectPackage(srcFiles) if moxieRoot := findMoxieRoot(); moxieRoot != "" { cmdBuildPerPkg(t, meshPath, outPath, optLevel, targetDir, target, moxieRoot, srcFiles, pkgName, modPath) return } fmt.Fprintln(os.Stderr, "compiling " | fmt.Sprint(len(srcFiles)) | " files from " | targetDir | " (package " | pkgName | ")") if modPath != "" { fmt.Fprintln(os.Stderr, "module: " | modPath) } result := iskra.CompileFiles(t, srcFiles, "", pkgName, true, modPath) fmt.Fprintln(os.Stderr, "matched: " | fmt.Sprint(result.Matched)) fmt.Fprintln(os.Stderr, "unmatched: " | fmt.Sprint(result.Unmatched)) if result.Unmatched > 0 { for _, u := range result.UnmatchedList { fmt.Fprintln(os.Stderr, "UNMATCHED: " | u.Name | " from " | u.SrcFile) } expanded := expandLattice(t, meshPath, result.UnmatchedList, targetDir, pkgName, modPath, target) if !expanded { fatal("lattice expansion failed - moxie not on PATH or -internal-printir failed") } result = iskra.CompileFiles(t, srcFiles, "", pkgName, true, modPath) fmt.Fprintln(os.Stderr, "after expansion: matched=" | fmt.Sprint(result.Matched) | " unmatched=" | fmt.Sprint(result.Unmatched)) if result.Unmatched > 0 { for _, u := range result.UnmatchedList { fmt.Fprintln(os.Stderr, "STILL UNMATCHED: " | u.Name) } fatal("lattice expansion did not resolve all functions - algorithm bug") } } if len(result.IR) == 0 { fatal("no functions compiled") } result.IR = injectDepFunctions(result.IR, targetDir, target) tmpDir, err := os.MkdirTemp("", "moxie-iskra-") if err != nil { fatal("creating temp dir: " | err.Error()) } llFile := filepath.Join(tmpDir, "module.ll") err = os.WriteFile(llFile, result.IR, 0644) if err != nil { fatal("writing IR: " | err.Error()) } fmt.Fprintln(os.Stderr, "wrote IR: " | llFile | " (" | fmt.Sprint(len(result.IR)) | " bytes)") bcFile := filepath.Join(tmpDir, "module.bc") if rc := runTool("opt", "-O" | optLevel, llFile, "-o", bcFile); rc != 0 { fatal("opt failed (exit " | fmt.Sprint(rc) | ")") } stdlibBC := resolveStdlibBitcode(target) useLTO := stdlibBC != "" && target != "wasm32" objFile := filepath.Join(tmpDir, "module.o") if useLTO { combinedBC := filepath.Join(tmpDir, "combined.bc") if rc := runTool("llvm-link", stdlibBC, "--override", bcFile, "-o", combinedBC); rc != 0 { fatal("llvm-link failed (exit " | fmt.Sprint(rc) | ")") } keepFile := filepath.Join(tmpDir, "keep.txt") keepSyms := "main\n_start\n__alloc\nmoxie_handle_fatal_signal\nmoxie_runtime_bdwgc_callback\nmoxie_scanstack\nmoxie_signal_handler\n" if target == "wasm32" { keepSyms = keepSyms | collectWasmExports(result.IR) } os.WriteFile(keepFile, []byte(keepSyms), 0644) optBC := filepath.Join(tmpDir, "combined-opt.bc") if rc := runTool("opt", "--passes=internalize,globaldce,default", "--internalize-public-api-file=" | keepFile, combinedBC, "-o", optBC); rc != 0 { fatal("opt LTO failed (exit " | fmt.Sprint(rc) | ")") } llcArgs := llcFlags(target, optLevel, optBC, objFile) if rc := runToolArgv(llcArgs); rc != 0 { fatal("llc failed (exit " | fmt.Sprint(rc) | ")") } fmt.Fprintln(os.Stderr, "LTO: linked module + stdlib bitcode") } else { llcArgs := llcFlags(target, optLevel, bcFile, objFile) if rc := runToolArgv(llcArgs); rc != 0 { fatal("llc failed (exit " | fmt.Sprint(rc) | ")") } } intrinsicsLL := generateIntrinsics(result.IR, target) intrinsicsFile := filepath.Join(tmpDir, "intrinsics.ll") err = os.WriteFile(intrinsicsFile, intrinsicsLL, 0644) if err != nil { fatal("writing intrinsics: " | err.Error()) } intrinsicsObj := filepath.Join(tmpDir, "intrinsics.o") intrinsicClangArgs := intrinsicCompileFlags(target, intrinsicsFile, intrinsicsObj) if rc := runToolArgv(intrinsicClangArgs); rc != 0 { fatal("compiling intrinsics failed") } home, _ := os.UserHomeDir() hasDeps := len(readDepModules(targetDir)) > 0 linkArgs := linkFlagsLTO(target, objFile, intrinsicsObj, outPath, home, useLTO, target == "wasm32" && hasDeps) if rc := runToolArgv(linkArgs); rc != 0 { fatal("link failed (exit " | fmt.Sprint(rc) | ")") } fmt.Fprintln(os.Stderr, "built: " | outPath) } func cmdCompile() { meshPath := "" outPath := "" pkgFilter := "" var srcFiles []string i := 2 for i < len(os.Args) { switch os.Args[i] { case "-mesh": i++ if i >= len(os.Args) { fatal("-mesh requires argument") } meshPath = os.Args[i] case "-o": i++ if i >= len(os.Args) { fatal("-o requires argument") } outPath = os.Args[i] case "-pkg": i++ if i >= len(os.Args) { fatal("-pkg requires argument") } pkgFilter = os.Args[i] default: srcFiles = append(srcFiles, os.Args[i]) } i++ } meshPath = resolveMesh(meshPath, "x86_64") if len(srcFiles) == 0 { fatal("no source files specified") } t := loadMesh(meshPath) result := iskra.CompileFiles(t, srcFiles, pkgFilter, pkgFilter, false) fmt.Fprintln(os.Stderr, "matched: " | fmt.Sprint(result.Matched)) fmt.Fprintln(os.Stderr, "unmatched: " | fmt.Sprint(result.Unmatched)) if len(result.IR) == 0 { fatal("no functions compiled") } if outPath != "" { err := os.WriteFile(outPath, result.IR, 0644) if err != nil { fatal("writing output: " | err.Error()) } fmt.Fprintln(os.Stderr, "wrote: " | outPath) } else { os.Stdout.Write(result.IR) } } func cmdBootstrap() { meshPath := "" moxieRoot := "" i := 2 for i < len(os.Args) { switch os.Args[i] { case "-mesh": i++ if i >= len(os.Args) { fatal("-mesh requires argument") } meshPath = os.Args[i] case "-m": i++ if i >= len(os.Args) { fatal("-m requires argument") } moxieRoot = os.Args[i] default: fatal("unknown argument: " | os.Args[i]) } i++ } if moxieRoot == "" { moxieRoot = os.Getenv("MOXIEROOT") } if moxieRoot == "" { fatal("specify moxie root with -m or $MOXIEROOT") } meshPath = resolveMesh(meshPath, "x86_64") t := loadMesh(meshPath) mxFiles, cFiles, sFiles := iskra.FindRuntimeSources(moxieRoot) fmt.Fprintln(os.Stderr, "runtime sources: " | fmt.Sprint(len(mxFiles)) | " .mx, " | fmt.Sprint(len(cFiles)) | " .c, " | fmt.Sprint(len(sFiles)) | " .S") if len(mxFiles) == 0 { fatal("no runtime .mx files found in " | moxieRoot | "/src/runtime/") } result := iskra.CompileFiles(t, mxFiles, "runtime", "runtime", false) fmt.Fprintln(os.Stderr, "runtime matched: " | fmt.Sprint(result.Matched)) fmt.Fprintln(os.Stderr, "runtime unmatched: " | fmt.Sprint(result.Unmatched)) home, _ := os.UserHomeDir() iskraDir := filepath.Join(home, ".local", "share", "moxie-iskra") os.MkdirAll(iskraDir, 0755) tmpDir, err := os.MkdirTemp("", "moxie-iskra-bootstrap-") if err != nil { fatal("creating temp dir: " | err.Error()) } if len(result.IR) > 0 { llFile := filepath.Join(tmpDir, "runtime.ll") err = os.WriteFile(llFile, result.IR, 0644) if err != nil { fatal("writing runtime IR: " | err.Error()) } fmt.Fprintln(os.Stderr, "wrote runtime IR: " | llFile) bcFile := filepath.Join(tmpDir, "runtime.bc") if rc := runTool("opt", "-O2", llFile, "-o", bcFile); rc != 0 { fatal("opt on runtime failed (exit " | fmt.Sprint(rc) | ")") } rtObj := filepath.Join(tmpDir, "runtime_mx.o") if rc := runTool("llc", "-filetype=obj", "-relocation-model=pic", "-O2", bcFile, "-o", rtObj); rc != 0 { fatal("llc on runtime failed (exit " | fmt.Sprint(rc) | ")") } allObjs := []string{rtObj} for _, cf := range cFiles { base := filepath.Base(cf) out := filepath.Join(tmpDir, base | ".o") if rc := runTool("clang", "-c", "-O2", "-o", out, cf); rc != 0 { fmt.Fprintln(os.Stderr, "warning: failed to compile " | cf) continue } allObjs = append(allObjs, out) } for _, sf := range sFiles { base := filepath.Base(sf) out := filepath.Join(tmpDir, base | ".o") if rc := runTool("clang", "-c", "-o", out, sf); rc != 0 { fmt.Fprintln(os.Stderr, "warning: failed to assemble " | sf) continue } allObjs = append(allObjs, out) } runtimeOut := filepath.Join(iskraDir, "runtime.o") if len(allObjs) == 1 { data, err := os.ReadFile(allObjs[0]) if err != nil { fatal("reading runtime object: " | err.Error()) } err = os.WriteFile(runtimeOut, data, 0644) if err != nil { fatal("writing runtime.o: " | err.Error()) } } else { ldArgs := []string{"ld", "-r", "-o", runtimeOut} ldArgs = append(ldArgs, allObjs...) if rc := runToolArgv(ldArgs); rc != 0 { fatal("linking runtime objects failed") } } fmt.Fprintln(os.Stderr, "installed: " | runtimeOut) } else { fmt.Fprintln(os.Stderr, "warning: no runtime functions compiled") } } func runTool(name string, args ...string) int { argv := []string{name} argv = append(argv, args...) return runToolArgv(argv) } func runToolArgv(argv []string) int { name := argv[0] fullPath := which(name) if fullPath == "" { fmt.Fprintln(os.Stderr, "tool not found: " | name) return 127 } proc, err := os.StartProcess(fullPath, argv, &os.ProcAttr{ Env: os.Environ(), }) if err != nil { fmt.Fprintln(os.Stderr, "exec " | name | ": " | err.Error()) return 126 } var ws syscall.WaitStatus for { _, err = syscall.Wait4(proc.Pid, &ws, 0, nil) if err == nil { break } if err != syscall.EINTR { fmt.Fprintln(os.Stderr, "wait " | name | ": " | err.Error()) return 126 } } if ws.Exited() { return ws.ExitStatus() } return 1 } func which(name string) string { if bytes.IndexByte([]byte(name), '/') >= 0 { return name } pathEnv := os.Getenv("PATH") dirs := bytes.Split([]byte(pathEnv), []byte(":")) for _, d := range dirs { full := string(d) | "/" | name if _, err := os.Stat(full); err == nil { return full } } return "" } func detectPackage(srcFiles []string) string { for _, f := range srcFiles { data, err := os.ReadFile(f) if err != nil { continue } lines := bytes.Split(data, []byte("\n")) for _, line := range lines { line = bytes.TrimSpace(line) if bytes.HasPrefix(line, []byte("package ")) { return string(line[8:]) } } } return "main" } func expandLattice(t *iskra.Tree, meshPath string, unmatched []iskra.UnmatchedFunc, srcDir string, pkg string, modPath string, target string) bool { moxiePath := which("moxie") if moxiePath == "" { return false } tmpDir, err := os.MkdirTemp("", "moxie-iskra-expand-") if err != nil { return false } irFile := filepath.Join(tmpDir, "lowered.ll") shScript := filepath.Join(tmpDir, "run.sh") envPrefix := "" if target == "wasm32" { envPrefix = "GOOS=js GOARCH=wasm " } shContent := "#!/bin/sh\ncd " | srcDir | " && " | envPrefix | moxiePath | " build -internal-printir -o /dev/null . > " | irFile | " 2>/dev/null\n" err = os.WriteFile(shScript, []byte(shContent), 0755) if err != nil { return false } shPath := which("sh") runTool(shPath, shScript) moduleIR, err := os.ReadFile(irFile) if err != nil || len(moduleIR) == 0 { fmt.Fprintln(os.Stderr, "expand: moxie -internal-printir produced no output") return false } fmt.Fprintln(os.Stderr, "expand: got " | fmt.Sprint(len(moduleIR)) | " bytes of IR from moxie") irBlocks := splitIRFunctions(moduleIR) if len(irBlocks) == 0 { return false } unmatchedNames := map[string]bool{} for _, u := range unmatched { unmatchedNames[u.Name] = true } srcByFile := map[string][]byte{} added := 0 scaffold := iskra.ExtractModuleScaffold(moduleIR) scaffold = stripAttrGroupRefsAll(scaffold) if target != "wasm32" { scaffold = filterScaffoldForPkg(scaffold, pkg, modPath) } modMetaIdx := iskra.InsertSegment(t, iskra.StageIR, iskra.KindPkg, pkg | ".__module__", scaffold) for _, u := range unmatched { src, ok := srcByFile[u.SrcFile] if !ok { src, err = os.ReadFile(u.SrcFile) if err != nil { fmt.Fprintln(os.Stderr, "expand: cannot read " | u.SrcFile | ": " | err.Error()) continue } srcByFile[u.SrcFile] = src } var decl []byte for _, d := range iskra.SplitDecls(src) { if iskra.DeclName(d) == u.Name { decl = d break } } if len(decl) == 0 { fmt.Fprintln(os.Stderr, "expand: cannot find decl for " | u.Name | " in " | u.SrcFile) continue } astDump := iskra.GenAST(decl) if len(astDump) == 0 { fmt.Fprintln(os.Stderr, "expand: empty AST for " | u.Name) continue } irBlock := findIRBlock(irBlocks, u.Name, pkg) if len(irBlock) == 0 { fmt.Fprintln(os.Stderr, "expand: no IR found for " | u.Name) continue } irBlock = stripAttrGroupRefs(irBlock) irBlock = iskra.StripDebugMetadata(irBlock) funcName := pkg | "." | u.Name astIdx := iskra.InsertSegment(t, iskra.StageAST, iskra.KindFunc, funcName, astDump) irIdx := iskra.InsertSegment(t, iskra.StageIR, iskra.KindFunc, funcName, irBlock) t.AddAdj(astIdx, irIdx) t.AddAdj(irIdx, astIdx) t.AddAdj(irIdx, modMetaIdx) t.AddAdj(modMetaIdx, irIdx) fmt.Fprintln(os.Stderr, "EXPANDED: " | funcName) added++ } if added == 0 { return false } t.FinalizeAdj() if err := iskra.MeshSaveFile(meshPath, t, iskra.StageAST, iskra.StageIR); err != nil { fmt.Fprintln(os.Stderr, "expand: saving mesh: " | err.Error()) return false } fmt.Fprintln(os.Stderr, "expand: added " | fmt.Sprint(added) | " entries, mesh saved to " | meshPath) return true } func splitIRFunctions(ir []byte) map[string][]byte { result := map[string][]byte{} lines := bytes.Split(ir, []byte("\n")) i := 0 for i < len(lines) { trimmed := bytes.TrimSpace(lines[i]) if !bytes.HasPrefix(trimmed, []byte("define ")) { i++ continue } name := iskra.ExtractAtName(trimmed) var funcLines []byte depth := 0 for i < len(lines) { funcLines = append(funcLines, lines[i]...) funcLines = append(funcLines, '\n') for _, ch := range string(lines[i]) { if ch == '{' { depth++ } else if ch == '}' { depth-- } } i++ if depth == 0 { break } } if name != "" { result[iskra.NormalizeLLVMName(name)] = funcLines } } return result } func readDepModules(dir string) []string { modFile := filepath.Join(dir, "moxie.mod") data, err := os.ReadFile(modFile) if err != nil { return nil } var deps []string lines := bytes.Split(data, []byte("\n")) for _, line := range lines { line = bytes.TrimSpace(line) if bytes.HasPrefix(line, []byte("require ")) { parts := bytes.Fields(line) if len(parts) >= 2 { deps = append(deps, string(parts[1])) } } } return deps } func collectDefinedNames(ir []byte) map[string]bool { names := map[string]bool{} lines := bytes.Split(ir, []byte("\n")) for _, line := range lines { trimmed := bytes.TrimSpace(line) if bytes.HasPrefix(trimmed, []byte("define ")) { name := iskra.ExtractAtName(trimmed) if name != "" { names[iskra.NormalizeLLVMName(name)] = true } } } return names } func injectDepFunctions(resultIR []byte, srcDir string, target string) []byte { if target != "wasm32" { return resultIR } deps := readDepModules(srcDir) if len(deps) == 0 { fmt.Fprintln(os.Stderr, "inject-deps: no dependency modules in moxie.mod") return resultIR } fmt.Fprintln(os.Stderr, "inject-deps: dependency modules: " | fmt.Sprint(len(deps))) moxiePath := which("moxie") if moxiePath == "" { fmt.Fprintln(os.Stderr, "inject-deps: moxie not on PATH") return resultIR } tmpDir, err := os.MkdirTemp("", "moxie-iskra-deps-") if err != nil { return resultIR } irFile := filepath.Join(tmpDir, "full.ll") shScript := filepath.Join(tmpDir, "run.sh") shContent := "#!/bin/sh\ncd " | srcDir | " && GOOS=js GOARCH=wasm " | moxiePath | " build -internal-printir -o /dev/null . > " | irFile | " 2>/dev/null\n" os.WriteFile(shScript, []byte(shContent), 0755) shPath := which("sh") runTool(shPath, shScript) fullIR, err := os.ReadFile(irFile) if err != nil || len(fullIR) == 0 { fmt.Fprintln(os.Stderr, "inject-deps: moxie -internal-printir produced no output") return resultIR } blocks := splitIRFunctions(fullIR) defined := collectDefinedNames(resultIR) injected := 0 var toInject []string for name := range blocks { if defined[name] { continue } toInject = append(toInject, name) } stubNames := map[string]bool{} for _, name := range toInject { stubNames[name] = true } resultIR = removeStubDeclarations(resultIR, stubNames) wasmExportAttrs := parseWasmExportAttrs(fullIR) var cleanedBlocks [][]byte var usedPtrs []string for _, name := range toInject { block := blocks[name] block = inlineWasmExportAttr(block, wasmExportAttrs) block = stripAttrGroupRefs(block) block = iskra.StripDebugMetadata(block) cleanedBlocks = append(cleanedBlocks, block) if bytes.Contains([]byte(name), []byte("#wasmexport")) { usedPtrs = append(usedPtrs, name) } } depGlobals := extractMissingGlobals(fullIR, resultIR, cleanedBlocks) if len(depGlobals) > 0 { resultIR = append(resultIR, '\n') resultIR = append(resultIR, depGlobals...) } if len(usedPtrs) > 0 { resultIR = stripLlvmUsed(resultIR) resultIR = appendLlvmUsed(resultIR, usedPtrs) } for _, block := range cleanedBlocks { resultIR = append(resultIR, '\n') resultIR = append(resultIR, block...) injected++ } depDecls := extractMissingDeclarations(fullIR, resultIR) if len(depDecls) > 0 { resultIR = append(resultIR, '\n') resultIR = append(resultIR, depDecls...) } fmt.Fprintln(os.Stderr, "inject-deps: added " | fmt.Sprint(injected) | " dependency functions") return resultIR } func extractMissingGlobals(fullIR []byte, moduleIR []byte, injectedBlocks [][]byte) []byte { existing := map[string]bool{} modLines := bytes.Split(moduleIR, []byte("\n")) for _, line := range modLines { trimmed := bytes.TrimSpace(line) if len(trimmed) > 0 && trimmed[0] == '@' { gname := iskra.NormalizeLLVMName(extractGlobalName(trimmed)) if gname != "" { existing[gname] = true } } if bytes.HasPrefix(trimmed, []byte("declare ")) || bytes.HasPrefix(trimmed, []byte("define ")) { name := iskra.ExtractAtName(trimmed) if name != "" { existing[iskra.NormalizeLLVMName(name)] = true } } } needed := map[string]bool{} for _, block := range injectedBlocks { collectGlobalRefs(block, needed) } allGlobals := map[string][]byte{} fullLines := bytes.Split(fullIR, []byte("\n")) for _, line := range fullLines { trimmed := bytes.TrimSpace(line) if len(trimmed) == 0 || trimmed[0] != '@' { continue } if bytes.Contains(trimmed, []byte(" = ")) && !bytes.HasPrefix(trimmed, []byte("@llvm.")) { gname := extractGlobalName(trimmed) if gname != "" { norm := iskra.NormalizeLLVMName(gname) if _, ok := allGlobals[norm]; !ok { allGlobals[norm] = trimmed } } } } var out []byte added := 0 queue := []string{} for ref := range needed { if !existing[ref] { queue = append(queue, ref) } } for len(queue) > 0 { ref := queue[len(queue)-1] queue = queue[:len(queue)-1] if existing[ref] { continue } existing[ref] = true gline, ok := allGlobals[ref] if !ok { continue } cleaned := stripAttrGroupRefs(iskra.StripDebugMetadata(gline)) out = append(out, cleaned...) out = append(out, '\n') added++ transitive := map[string]bool{} collectGlobalRefs(gline, transitive) for tr := range transitive { if !existing[tr] { queue = append(queue, tr) } } } if added > 0 { fmt.Fprintln(os.Stderr, "inject-deps: added " | fmt.Sprint(added) | " missing globals") } return out } func parseWasmExportAttrs(ir []byte) map[string]string { result := map[string]string{} lines := bytes.Split(ir, []byte("\n")) for _, line := range lines { trimmed := bytes.TrimSpace(line) if !bytes.HasPrefix(trimmed, []byte("attributes #")) { continue } nameIdx := bytes.Index(trimmed, []byte("\"wasm-export-name\"=\"")) if nameIdx < 0 { continue } hashStart := len("attributes ") spaceIdx := bytes.IndexByte(trimmed[hashStart:], ' ') if spaceIdx < 0 { continue } attrID := string(trimmed[hashStart : hashStart+spaceIdx]) valStart := nameIdx + len("\"wasm-export-name\"=\"") rest := trimmed[valStart:] closeQuote := bytes.IndexByte(rest, '"') if closeQuote < 0 { continue } result[attrID] = string(rest[:closeQuote]) } return result } func inlineWasmExportAttr(block []byte, attrs map[string]string) []byte { lines := bytes.Split(block, []byte("\n")) if len(lines) == 0 { return block } firstLine := lines[0] if !bytes.HasPrefix(bytes.TrimSpace(firstLine), []byte("define ")) { return block } for attrID, exportName := range attrs { ref := []byte(" " | attrID | " ") refEnd := []byte(" " | attrID | "\n") refBrace := []byte(") " | attrID | " {") if bytes.Contains(firstLine, ref) || bytes.HasSuffix(firstLine, refEnd) || bytes.Contains(firstLine, refBrace) { inlineAttr := []byte(" \"wasm-export-name\"=\"" | exportName | "\"") braceIdx := bytes.LastIndexByte(firstLine, '{') if braceIdx > 0 { var newLine []byte newLine = append(newLine, firstLine[:braceIdx]...) newLine = append(newLine, inlineAttr...) newLine = append(newLine, " {"...) lines[0] = newLine return bytes.Join(lines, []byte("\n")) } } } return block } func stripLlvmUsed(ir []byte) []byte { prefix := []byte("@llvm.used = ") lines := bytes.Split(ir, []byte("\n")) var out []byte for i, line := range lines { if bytes.HasPrefix(bytes.TrimSpace(line), prefix) { continue } if i > 0 { out = append(out, '\n') } out = append(out, line...) } return out } func appendLlvmUsed(ir []byte, ptrs []string) []byte { marker := []byte("@llvm.used = appending global [") existing := bytes.Index(ir, marker) if existing >= 0 { lineEnd := bytes.IndexByte(ir[existing:], '\n') if lineEnd < 0 { lineEnd = len(ir) - existing } line := ir[existing : existing+lineEnd] lastBracket := bytes.LastIndexByte(line, ']') if lastBracket > 0 { var extra []byte newCount := 0 for _, p := range ptrs { name := p if len(name) > 0 && name[0] == '@' { if len(name) > 1 && name[1] != '"' { name = "@\"" | name[1:] | "\"" } } if !bytes.Contains(line, []byte(name)) { extra = append(extra, ", ptr "...) extra = append(extra, name...) newCount++ } } if newCount > 0 { countStart := len(marker) countEnd := bytes.IndexByte(line[countStart:], ' ') if countEnd > 0 { oldCountStr := string(line[countStart : countStart+countEnd]) oldCount := 0 for _, ch := range oldCountStr { if ch >= '0' && ch <= '9' { oldCount = oldCount*10 + int(ch-'0') } } totalCount := oldCount + newCount newCountStr := fmt.Sprint(totalCount) var newLine []byte newLine = append(newLine, line[:countStart]...) newLine = append(newLine, newCountStr...) newLine = append(newLine, line[countStart+countEnd:]...) newLastBracket := bytes.LastIndexByte(newLine, ']') var merged []byte merged = append(merged, newLine[:newLastBracket]...) merged = append(merged, extra...) merged = append(merged, newLine[newLastBracket:]...) var result []byte result = append(result, ir[:existing]...) result = append(result, merged...) result = append(result, ir[existing+lineEnd:]...) return result } } return ir } } ir = append(ir, '\n') entry := "@llvm.used = appending global [" | fmt.Sprint(len(ptrs)) | " x ptr] [" for i, p := range ptrs { if i > 0 { entry = entry | ", " } name := p if len(name) > 0 && name[0] == '@' { if len(name) > 1 && name[1] != '"' { name = "@\"" | name[1:] | "\"" } } entry = entry | "ptr " | name } entry = entry | "]\n" ir = append(ir, []byte(entry)...) return ir } func collectWasmExports(ir []byte) string { var exports string lines := bytes.Split(ir, []byte("\n")) for _, line := range lines { trimmed := bytes.TrimSpace(line) if !bytes.HasPrefix(trimmed, []byte("define ")) { continue } name := iskra.ExtractAtName(trimmed) if name == "" { continue } norm := iskra.NormalizeLLVMName(name) if bytes.Contains([]byte(norm), []byte("#wasmexport")) { bare := norm if len(bare) > 0 && bare[0] == '@' { bare = bare[1:] } if len(bare) > 0 && bare[0] == '"' { bare = bare[1:] } if len(bare) > 0 && bare[len(bare)-1] == '"' { bare = bare[:len(bare)-1] } exports = exports | bare | "\n" } } return exports } func defineToDecl(line []byte) []byte { braceIdx := bytes.LastIndexByte(line, '{') if braceIdx < 0 { return nil } sig := bytes.TrimSpace(line[:braceIdx]) rest := sig[len("define "):] for { if bytes.HasPrefix(rest, []byte("internal ")) { rest = rest[len("internal "):] } else if bytes.HasPrefix(rest, []byte("hidden ")) { rest = rest[len("hidden "):] } else if bytes.HasPrefix(rest, []byte("linkonce_odr ")) { rest = rest[len("linkonce_odr "):] } else if bytes.HasPrefix(rest, []byte("unnamed_addr ")) { rest = rest[len("unnamed_addr "):] } else { break } } parenClose := bytes.LastIndexByte(rest, ')') if parenClose > 0 { rest = rest[:parenClose+1] } var decl []byte decl = append(decl, "declare "...) decl = append(decl, rest...) return decl } func extractMissingDeclarations(fullIR []byte, moduleIR []byte) []byte { existing := map[string]bool{} modLines := bytes.Split(moduleIR, []byte("\n")) for _, line := range modLines { trimmed := bytes.TrimSpace(line) if bytes.HasPrefix(trimmed, []byte("define ")) || bytes.HasPrefix(trimmed, []byte("declare ")) { name := iskra.ExtractAtName(trimmed) if name != "" { existing[iskra.NormalizeLLVMName(name)] = true } } } refs := map[string]bool{} for _, line := range modLines { collectGlobalRefs(line, refs) } allDecls := map[string][]byte{} fullLines := bytes.Split(fullIR, []byte("\n")) for _, line := range fullLines { trimmed := bytes.TrimSpace(line) if bytes.HasPrefix(trimmed, []byte("declare ")) { name := iskra.ExtractAtName(trimmed) if name != "" { norm := iskra.NormalizeLLVMName(name) if _, ok := allDecls[norm]; !ok { allDecls[norm] = trimmed } } } } allDefineToDecl := map[string][]byte{} i := 0 for i < len(fullLines) { trimmed := bytes.TrimSpace(fullLines[i]) if bytes.HasPrefix(trimmed, []byte("define ")) { name := iskra.ExtractAtName(trimmed) if name != "" { norm := iskra.NormalizeLLVMName(name) if _, ok := allDecls[norm]; !ok { if _, ok2 := allDefineToDecl[norm]; !ok2 { decl := defineToDecl(trimmed) if len(decl) > 0 { allDefineToDecl[norm] = decl } } } } depth := 0 for i < len(fullLines) { for _, ch := range string(fullLines[i]) { if ch == '{' { depth++ } else if ch == '}' { depth-- } } i++ if depth == 0 { break } } continue } i++ } var out []byte added := 0 for ref := range refs { if existing[ref] { continue } if dline, ok := allDecls[ref]; ok { cleaned := stripAttrGroupRefs(iskra.StripDebugMetadata(dline)) out = append(out, cleaned...) out = append(out, '\n') existing[ref] = true added++ } else if dline, ok := allDefineToDecl[ref]; ok { cleaned := stripAttrGroupRefs(iskra.StripDebugMetadata(dline)) out = append(out, cleaned...) out = append(out, '\n') existing[ref] = true added++ } } if added > 0 { fmt.Fprintln(os.Stderr, "inject-deps: added " | fmt.Sprint(added) | " missing declarations") } return out } func collectGlobalRefs(data []byte, refs map[string]bool) { i := 0 for i < len(data) { atIdx := bytes.IndexByte(data[i:], '@') if atIdx < 0 { break } atIdx += i name := extractGlobalName(data[atIdx:]) if name != "" && !bytes.HasPrefix([]byte(name), []byte("@llvm.")) { refs[iskra.NormalizeLLVMName(name)] = true } i = atIdx + 1 } } func extractGlobalName(line []byte) string { if len(line) == 0 || line[0] != '@' { return "" } if len(line) > 1 && line[1] == '"' { closeQuote := bytes.IndexByte(line[2:], '"') if closeQuote < 0 { return "" } return string(line[:closeQuote+3]) } for i := 1; i < len(line); i++ { ch := line[i] if ch == ' ' || ch == '=' || ch == '(' || ch == ',' || ch == ')' || ch == '\n' || ch == '\r' { return string(line[:i]) } } return string(line) } func removeStubDeclarations(ir []byte, names map[string]bool) []byte { lines := bytes.Split(ir, []byte("\n")) var out []byte removed := 0 for _, line := range lines { trimmed := bytes.TrimSpace(line) if bytes.HasPrefix(trimmed, []byte("declare ")) { name := iskra.ExtractAtName(trimmed) if name != "" && names[iskra.NormalizeLLVMName(name)] { removed++ continue } } out = append(out, line...) out = append(out, '\n') } if removed > 0 { fmt.Fprintln(os.Stderr, "inject-deps: removed " | fmt.Sprint(removed) | " stub declarations") } return out } func findIRBlock(blocks map[string][]byte, funcName string, pkg string) []byte { candidates := []string{ "@" | pkg | "." | funcName, "@\"" | pkg | "." | funcName | "\"", } for _, c := range candidates { if b, ok := blocks[c]; ok { return b } } suffix := "." | funcName pkgDot := pkg | "." for name, b := range blocks { if bytes.HasSuffix([]byte(name), []byte(suffix)) && bytes.Contains([]byte(name), []byte(pkgDot)) { return b } } return nil } func filterScaffoldForPkg(scaffold []byte, pkg string, modPath string) []byte { lines := bytes.Split(scaffold, []byte("\n")) var out []byte modPrefix := []byte(modPath | "/") pkgDot := []byte(pkg | ".") pkgDollar := []byte(pkg | "$") isAppPkg := func(name []byte) bool { return bytes.Contains(name, pkgDot) || bytes.Contains(name, pkgDollar) || bytes.Contains(name, modPrefix) } for _, line := range lines { trimmed := bytes.TrimSpace(line) if len(trimmed) == 0 { out = append(out, '\n') continue } if bytes.HasPrefix(trimmed, []byte("target ")) || bytes.HasPrefix(trimmed, []byte("source_filename")) { out = append(out, line...) out = append(out, '\n') continue } if trimmed[0] == '%' { out = append(out, line...) out = append(out, '\n') continue } if trimmed[0] == '@' { gname := trimmed spIdx := bytes.IndexByte(gname, ' ') if spIdx > 0 { gname = gname[:spIdx] } if isAppPkg(gname) { out = append(out, line...) out = append(out, '\n') } continue } if bytes.HasPrefix(trimmed, []byte("declare")) { if isAppPkg(trimmed) || bytes.Contains(trimmed, []byte("wasm-import-module")) { out = append(out, line...) out = append(out, '\n') } continue } out = append(out, line...) out = append(out, '\n') } return out } func stripAttrGroupRefsAll(ir []byte) []byte { lines := bytes.Split(ir, []byte("\n")) wasmAttrs := map[string]string{} for _, line := range lines { trimmed := bytes.TrimSpace(line) if !bytes.HasPrefix(trimmed, []byte("attributes #")) { continue } if !bytes.Contains(trimmed, []byte("wasm-import-module")) { continue } idx := bytes.Index(trimmed, []byte("#")) end := idx + 1 for end < len(trimmed) && trimmed[end] >= '0' && trimmed[end] <= '9' { end++ } groupID := string(trimmed[idx:end]) var attrs []byte if m := extractQuotedAttr(trimmed, "wasm-import-module"); m != "" { attrs = append(attrs, " \"wasm-import-module\"=\""...) attrs = append(attrs, m...) attrs = append(attrs, '"') } if n := extractQuotedAttr(trimmed, "wasm-import-name"); n != "" { attrs = append(attrs, " \"wasm-import-name\"=\""...) attrs = append(attrs, n...) attrs = append(attrs, '"') } if len(attrs) > 0 { wasmAttrs[groupID] = string(attrs) } } var out []byte for _, line := range lines { trimmed := bytes.TrimSpace(line) if bytes.HasPrefix(trimmed, []byte("attributes #")) { continue } if len(wasmAttrs) > 0 && bytes.HasPrefix(trimmed, []byte("declare ")) { idx := bytes.LastIndex(line, []byte(" #")) if idx >= 0 { end := idx + 2 if end < len(line) && line[end] >= '0' && line[end] <= '9' { for end < len(line) && line[end] >= '0' && line[end] <= '9' { end++ } ref := string(line[idx+1 : end]) if attrs, ok := wasmAttrs[ref]; ok { var newLine []byte newLine = append(newLine, line[:idx]...) newLine = append(newLine, attrs...) newLine = append(newLine, line[end:]...) line = newLine } } } } line = stripAttrGroupRefsLine(line) out = append(out, line...) out = append(out, '\n') } return out } func extractQuotedAttr(line []byte, key string) string { needle := []byte("\"" | key | "\"=\"") idx := bytes.Index(line, needle) if idx < 0 { return "" } start := idx + len(needle) end := bytes.IndexByte(line[start:], '"') if end < 0 { return "" } return string(line[start : start+end]) } func stripAttrGroupRefsLine(line []byte) []byte { limit := len(line) if cIdx := bytes.Index(line, []byte(" c\"")); cIdx >= 0 { limit = cIdx } for { sub := line[:limit] idx := bytes.LastIndex(sub, []byte(" #")) if idx < 0 { break } end := idx + 2 if end >= len(line) || line[end] < '0' || line[end] > '9' { break } for end < len(line) && line[end] >= '0' && line[end] <= '9' { end++ } var newLine []byte newLine = append(newLine, line[:idx]...) newLine = append(newLine, line[end:]...) line = newLine limit = idx } return line } func stripAttrGroupRefs(ir []byte) []byte { nl := bytes.IndexByte(ir, '\n') if nl < 0 { return ir } first := ir[:nl] rest := ir[nl:] for { idx := bytes.LastIndex(first, []byte(" #")) if idx < 0 { break } end := idx + 2 if end >= len(first) || first[end] < '0' || first[end] > '9' { break } for end < len(first) && first[end] >= '0' && first[end] <= '9' { end++ } var newFirst []byte newFirst = append(newFirst, first[:idx]...) newFirst = append(newFirst, first[end:]...) first = newFirst } var out []byte out = append(out, first...) out = append(out, rest...) return out } func generateIntrinsics(ir []byte, target string) []byte { lines := bytes.Split(ir, []byte("\n")) var out []byte dl, triple := targetDataLayout(target) out = append(out, []byte("target datalayout = \"" | dl | "\"\n")...) out = append(out, []byte("target triple = \"" | triple | "\"\n\n")...) needTypeAssert := false for _, line := range lines { trimmed := bytes.TrimSpace(line) if bytes.HasPrefix(trimmed, []byte("@\"reflect/types.typeid:")) && bytes.Contains(trimmed, []byte("= external constant i8")) { nameEnd := bytes.Index(trimmed, []byte("\" =")) if nameEnd > 0 { name := trimmed[:nameEnd+1] out = append(out, name...) out = append(out, []byte(" = constant i8 0\n")...) } continue } if bytes.HasPrefix(trimmed, []byte("declare")) && bytes.Contains(trimmed, []byte("@runtime.typeAssert")) { needTypeAssert = true continue } if bytes.HasPrefix(trimmed, []byte("declare")) && bytes.Contains(trimmed, []byte("@\"interface:")) { name := iskra.ExtractAtName(trimmed) if name != "" { norm := iskra.NormalizeLLVMName(name) if bytes.HasSuffix([]byte(norm), []byte(".$typeassert")) { out = append(out, []byte("define i1 " | name | "(ptr %0, ptr %1) {\n ret i1 false\n}\n")...) } else if bytes.HasSuffix([]byte(norm), []byte("$invoke")) { out = append(out, []byte("define void " | name | "(ptr %0, ptr %1, ptr %2) {\n unreachable\n}\n")...) } } continue } if target == "x86_64" && bytes.HasPrefix(trimmed, []byte("@\"internal/cpu.X86\"")) && bytes.Contains(trimmed, []byte("= external")) { out = append(out, []byte("@\"internal/cpu.X86\" = global [1 x i8] zeroinitializer\n")...) continue } } if needTypeAssert { out = append(out, []byte("define i1 @runtime.typeAssert(ptr %actual, ptr %expected, ptr %context) {\n %cmp = icmp eq ptr %actual, %expected\n ret i1 %cmp\n}\n")...) } return out } func targetDataLayout(target string) (string, string) { if target == "wasm32" { return "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20", "wasm32-unknown-js" } return "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", "x86_64-unknown-linux-musleabihf" } func llcFlags(target string, optLevel string, input string, output string) []string { if target == "wasm32" { return []string{"llc", "-filetype=obj", "-mtriple=wasm32-unknown-js", "-O" | optLevel, input, "-o", output} } return []string{"llc", "-filetype=obj", "-relocation-model=pic", "-O" | optLevel, input, "-o", output} } func intrinsicCompileFlags(target string, input string, output string) []string { if target == "wasm32" { return []string{"clang", "-c", "--target=wasm32-unknown-js", "-o", output, input} } return []string{"clang", "-c", "-o", output, input} } func linkFlagsLTO(target string, objFile string, intrinsicsObj string, outPath string, home string, lto bool, skipStdlib bool) []string { if target == "wasm32" { args := []string{"wasm-ld", objFile, intrinsicsObj} runtimeObj := filepath.Join(home, ".local", "share", "moxie-iskra", "runtime.wasm32.o") if _, err := os.Stat(runtimeObj); err == nil { args = append(args, runtimeObj) fmt.Fprintln(os.Stderr, "linking with runtime: " | runtimeObj) } if !lto && !skipStdlib { stdlibArchive := resolveStdlibArchive(target) if stdlibArchive != "" { args = append(args, stdlibArchive) fmt.Fprintln(os.Stderr, "linking with stdlib: " | stdlibArchive) } } args = append(args, "--allow-undefined", "--gc-sections", "--no-entry", "--export=__alloc", "--export=_start", "--export=memory", "-o", outPath) return args } args := []string{"clang", objFile, intrinsicsObj} runtimeObj := filepath.Join(home, ".local", "share", "moxie-iskra", "runtime.o") if _, err := os.Stat(runtimeObj); err == nil { args = append(args, runtimeObj) fmt.Fprintln(os.Stderr, "linking with runtime: " | runtimeObj) } if lto { bdwgcObj := filepath.Join(home, ".local", "share", "moxie-iskra", "bdwgc.o") if _, err := os.Stat(bdwgcObj); err == nil { args = append(args, bdwgcObj) } asmStubsObj := filepath.Join(home, ".local", "share", "moxie-iskra", "asm_stubs.o") if _, err := os.Stat(asmStubsObj); err == nil { args = append(args, asmStubsObj) } } else { stdlibArchive := resolveStdlibArchive(target) if stdlibArchive != "" { args = append(args, stdlibArchive) fmt.Fprintln(os.Stderr, "linking with stdlib: " | stdlibArchive) } } args = append(args, "-no-pie", "-o", outPath, "-lm", "-Wl,--gc-sections") return args } func resolveStdlibArchive(target string) string { home, _ := os.UserHomeDir() if target == "wasm32" { wasmCorpus := os.Getenv("ISKRA_CORPUS_WASM") if wasmCorpus == "" { wasmCorpus = "/tmp/iskra-corpus-wasm" } archive := filepath.Join(wasmCorpus, "libstdlib.a") if _, err := os.Stat(archive); err == nil { return archive } if home != "" { archive = filepath.Join(home, ".local", "share", "moxie-iskra", "libstdlib.wasm32.a") if _, err := os.Stat(archive); err == nil { return archive } } return "" } corpusDir := os.Getenv("ISKRA_CORPUS") if corpusDir == "" { corpusDir = "/tmp/iskra-corpus" } archive := filepath.Join(corpusDir, "libstdlib.a") if _, err := os.Stat(archive); err == nil { return archive } if home != "" { archive = filepath.Join(home, ".local", "share", "moxie-iskra", "libstdlib.a") if _, err := os.Stat(archive); err == nil { return archive } } return "" } func resolveStdlibBitcode(target string) string { home, _ := os.UserHomeDir() if home == "" { return "" } bc := filepath.Join(home, ".local", "share", "moxie-iskra", "stdlib.bc") if target != "" && target != "x86_64" { bc = filepath.Join(home, ".local", "share", "moxie-iskra", "stdlib." | target | ".bc") } if _, err := os.Stat(bc); err == nil { return bc } return "" } func fatal(msg string) { fmt.Fprintln(os.Stderr, "moxie-iskra: " | msg) os.Exit(1) } // --- per-package compilation pipeline --- type pkgSpec struct { importPath string srcDir string pkgName string } func findMoxieRoot() string { if r := os.Getenv("MOXIEROOT"); r != "" { return r } p := which("moxie") if p == "" { return "" } // ask moxie env for MOXIEROOT tmpDir, err := os.MkdirTemp("", "moxie-iskra-env-") if err == nil { outFile := filepath.Join(tmpDir, "env.txt") shScript := filepath.Join(tmpDir, "run.sh") os.WriteFile(shScript, []byte("#!/bin/sh\n" | p | " env > " | outFile | "\n"), 0755) runTool(which("sh"), shScript) if data, err := os.ReadFile(outFile); err == nil { for _, line := range bytes.Split(data, []byte("\n")) { if bytes.HasPrefix(line, []byte("MOXIEROOT=")) { val := string(bytes.Trim(line[10:], "\"")) if val != "" { return val } } } } } // fallback: look for src/ sibling to binary dir := filepath.Dir(p) if filepath.Base(dir) == "bin" { dir = filepath.Dir(dir) } if _, err := os.Stat(filepath.Join(dir, "src")); err == nil { return dir } return "" } func pkgFileSafe(importPath string) string { b := []byte(importPath) for i := 0; i < len(b); i++ { if b[i] == '/' { b[i] = '_' } } return string(b) } func extractImportStr(data []byte) string { start := bytes.IndexByte(data, '"') if start < 0 { return "" } end := bytes.IndexByte(data[start+1:], '"') if end < 0 { return "" } return string(data[start+1 : start+1+end]) } func parseFileImports(data []byte) []string { var out []string lines := bytes.Split(data, []byte("\n")) inBlock := false for _, line := range lines { t := bytes.TrimSpace(line) if bytes.Equal(t, []byte("import (")) { inBlock = true continue } if inBlock { if len(t) > 0 && t[0] == ')' { inBlock = false continue } if s := extractImportStr(t); s != "" { out = append(out, s) } continue } if bytes.HasPrefix(t, []byte("import \"")) { if s := extractImportStr(t[7:]); s != "" { out = append(out, s) } } } return out } func isBuiltinPkg(imp string) bool { switch imp { case "moxie", "unsafe", "C": return true } return false } func transitiveDeps(initial []string, moxieRoot string) []pkgSpec { seen := map[string]bool{} var result []pkgSpec var queue []string for _, imp := range initial { queue = append(queue, imp) } for len(queue) > 0 { imp := queue[0] queue = queue[1:] if seen[imp] || isBuiltinPkg(imp) { continue } seen[imp] = true srcDir := filepath.Join(moxieRoot, "src", imp) if _, err := os.Stat(srcDir); err != nil { continue } srcFiles, err := iskra.DiscoverMxFiles(srcDir) if err != nil || len(srcFiles) == 0 { continue } pkgName := detectPackage(srcFiles) result = append(result, pkgSpec{importPath: imp, srcDir: srcDir, pkgName: pkgName}) for _, f := range srcFiles { data, _ := os.ReadFile(f) for _, sub := range parseFileImports(data) { if !seen[sub] { queue = append(queue, sub) } } } } return result } // pathMtime returns the mtime (seconds) of a path, 0 on error. func pathMtime(path string) int64 { fi, err := os.Stat(path) if err != nil { return 0 } return fi.ModTime().Unix() } // srcDirMtime returns the max mtime of any .mx file in srcDir. func srcDirMtime(srcDir string) int64 { files, err := iskra.DiscoverMxFiles(srcDir) if err != nil { return 0 } var maxMtime int64 for _, f := range files { if mt := pathMtime(f); mt > maxMtime { maxMtime = mt } } return maxMtime } // pkgCacheDir returns the cache directory for a package source dir. // Layout: $HOME/.local/share/moxie-iskra/cache/ func pkgCacheDir(home string, srcDir string) string { rel := srcDir if len(rel) > 0 && rel[0] == '/' { rel = rel[1:] } return filepath.Join(home, ".local", "share", "moxie-iskra", "cache", rel) } func pkgCacheObj(home string, srcDir string) string { return filepath.Join(pkgCacheDir(home, srcDir), "pkg.o") } func pkgCacheIR(home string, srcDir string) string { return filepath.Join(pkgCacheDir(home, srcDir), "pkg.ir") } // isCacheValid returns true if the cached .o exists and is newer than all source files. func isCacheValid(cacheObj string, srcDir string) bool { objMtime := pathMtime(cacheObj) if objMtime == 0 { return false } return objMtime >= srcDirMtime(srcDir) } // fetchAppIR runs moxie -internal-printir on the app and returns the IR bytes. func fetchAppIR(appSrcDir string, target string) []byte { moxiePath := which("moxie") if moxiePath == "" { return nil } tmpDir, err := os.MkdirTemp("", "moxie-iskra-appir-") if err != nil { return nil } irFile := filepath.Join(tmpDir, "app.ll") shScript := filepath.Join(tmpDir, "run.sh") envPrefix := "" if target == "wasm32" { envPrefix = "GOOS=js GOARCH=wasm " } shContent := "#!/bin/sh\ncd " | appSrcDir | " && " | envPrefix | moxiePath | " build -internal-printir -o /dev/null . > " | irFile | " 2>/dev/null\n" os.WriteFile(shScript, []byte(shContent), 0755) shPath := which("sh") runTool(shPath, shScript) data, err := os.ReadFile(irFile) if err != nil || len(data) == 0 { return nil } return data } // expandLatticeFromIR expands the lattice using pre-fetched app IR instead of re-running moxie. // Returns the number of new entries added. func expandLatticeFromIR(t *iskra.Tree, meshPath string, unmatched []iskra.UnmatchedFunc, moduleIR []byte, pkg string, modPath string, target string) int { if len(moduleIR) == 0 { return 0 } irBlocks := splitIRFunctions(moduleIR) scaffold := iskra.ExtractModuleScaffold(moduleIR) scaffold = stripAttrGroupRefsAll(scaffold) if target != "wasm32" { scaffold = filterScaffoldForPkg(scaffold, pkg, modPath) } modMetaIdx := iskra.InsertSegment(t, iskra.StageIR, iskra.KindPkg, pkg | ".__module__", scaffold) srcByFile := map[string][]byte{} added := 0 for _, u := range unmatched { src, ok := srcByFile[u.SrcFile] if !ok { var err error src, err = os.ReadFile(u.SrcFile) if err != nil { continue } srcByFile[u.SrcFile] = src } var decl []byte for _, d := range iskra.SplitDecls(src) { if iskra.DeclName(d) == u.Name { decl = d break } } if len(decl) == 0 { continue } astDump := iskra.GenAST(decl) if len(astDump) == 0 { continue } irBlock := findIRBlock(irBlocks, u.Name, pkg) if len(irBlock) == 0 { continue } irBlock = stripAttrGroupRefs(irBlock) irBlock = iskra.StripDebugMetadata(irBlock) funcName := pkg | "." | u.Name astIdx := iskra.InsertSegment(t, iskra.StageAST, iskra.KindFunc, funcName, astDump) irIdx := iskra.InsertSegment(t, iskra.StageIR, iskra.KindFunc, funcName, irBlock) t.AddAdj(astIdx, irIdx) t.AddAdj(irIdx, astIdx) t.AddAdj(irIdx, modMetaIdx) t.AddAdj(modMetaIdx, irIdx) fmt.Fprintln(os.Stderr, "EXPANDED: " | funcName) added++ } if added > 0 { t.FinalizeAdj() if err := iskra.MeshSaveFile(meshPath, t, iskra.StageAST, iskra.StageIR); err != nil { fmt.Fprintln(os.Stderr, "expand: saving mesh: " | err.Error()) } else { fmt.Fprintln(os.Stderr, "expand: added " | fmt.Sprint(added) | " entries, mesh saved") } } return added } // compilePkgToObj compiles a single package to a native object file. // Returns (objPath, irBytes). Uses the cache when valid; populates cache on miss. func compilePkgToObj(t *iskra.Tree, pkg pkgSpec, appIR []byte, meshPath string, target string, optLevel string, tmpDir string, home string) (string, []byte) { cacheObj := pkgCacheObj(home, pkg.srcDir) cacheIRFile := pkgCacheIR(home, pkg.srcDir) if isCacheValid(cacheObj, pkg.srcDir) { irBytes, err := os.ReadFile(cacheIRFile) if err == nil { fmt.Fprintln(os.Stderr, "pkg " | pkg.importPath | ": cached") return cacheObj, irBytes } } srcFiles, err := iskra.DiscoverMxFiles(pkg.srcDir) if err != nil || len(srcFiles) == 0 { return "", nil } modPath := iskra.ReadModulePath(pkg.srcDir) result := iskra.CompileFiles(t, srcFiles, "", pkg.pkgName, true, modPath) fmt.Fprintln(os.Stderr, "pkg " | pkg.importPath | ": matched=" | fmt.Sprint(result.Matched) | " unmatched=" | fmt.Sprint(result.Unmatched)) if result.Unmatched > 0 && len(appIR) > 0 { expandLatticeFromIR(t, meshPath, result.UnmatchedList, appIR, pkg.pkgName, modPath, target) result = iskra.CompileFiles(t, srcFiles, "", pkg.pkgName, true, modPath) if result.Unmatched > 0 { fmt.Fprintln(os.Stderr, "pkg " | pkg.importPath | ": " | fmt.Sprint(result.Unmatched) | " unmatched after expansion (unreachable, skipped)") } } if len(result.IR) == 0 { return "", nil } base := pkgFileSafe(pkg.importPath) llFile := filepath.Join(tmpDir, base | ".ll") bcFile := filepath.Join(tmpDir, base | ".bc") objFile := filepath.Join(tmpDir, base | ".o") if err := os.WriteFile(llFile, result.IR, 0644); err != nil { return "", nil } if rc := runTool("opt", "-O" | optLevel, llFile, "-o", bcFile); rc != 0 { fmt.Fprintln(os.Stderr, "pkg " | pkg.importPath | ": opt failed") return "", nil } llcArgs := llcFlags(target, optLevel, bcFile, objFile) if rc := runToolArgv(llcArgs); rc != 0 { fmt.Fprintln(os.Stderr, "pkg " | pkg.importPath | ": llc failed") return "", nil } // populate cache cacheDir := pkgCacheDir(home, pkg.srcDir) if err := os.MkdirAll(cacheDir, 0755); err == nil { objData, err := os.ReadFile(objFile) if err == nil { os.WriteFile(cacheObj, objData, 0644) os.WriteFile(cacheIRFile, result.IR, 0644) } } return objFile, result.IR } // linkMultiObj builds the linker command for multiple per-package object files. func linkMultiObj(target string, objs []string, outPath string, home string) []string { if target == "wasm32" { args := []string{"wasm-ld"} args = append(args, objs...) runtimeObj := filepath.Join(home, ".local", "share", "moxie-iskra", "runtime.wasm32.o") if _, err := os.Stat(runtimeObj); err == nil { args = append(args, runtimeObj) fmt.Fprintln(os.Stderr, "linking with runtime: " | runtimeObj) } args = append(args, "--allow-undefined", "--gc-sections", "--no-entry", "--export=__alloc", "--export=_start", "--export=memory", "-o", outPath) return args } args := []string{"clang"} args = append(args, objs...) runtimeObj := filepath.Join(home, ".local", "share", "moxie-iskra", "runtime.o") if _, err := os.Stat(runtimeObj); err == nil { args = append(args, runtimeObj) fmt.Fprintln(os.Stderr, "linking with runtime: " | runtimeObj) } bdwgcObj := filepath.Join(home, ".local", "share", "moxie-iskra", "bdwgc.o") if _, err := os.Stat(bdwgcObj); err == nil { args = append(args, bdwgcObj) } asmStubsObj := filepath.Join(home, ".local", "share", "moxie-iskra", "asm_stubs.o") if _, err := os.Stat(asmStubsObj); err == nil { args = append(args, asmStubsObj) } args = append(args, "-no-pie", "-o", outPath, "-lm", "-Wl,--gc-sections") return args } func cmdBuildPerPkg(t *iskra.Tree, meshPath string, outPath string, optLevel string, targetDir string, target string, moxieRoot string, srcFiles []string, pkgName string, modPath string) { fmt.Fprintln(os.Stderr, "per-package mode (moxie root: " | moxieRoot | ")") tmpDir, err := os.MkdirTemp("", "moxie-iskra-") if err != nil { fatal("creating temp dir: " | err.Error()) } home, _ := os.UserHomeDir() // collect direct imports from app source seen := map[string]bool{} var directImports []string for _, f := range srcFiles { data, _ := os.ReadFile(f) for _, imp := range parseFileImports(data) { if !seen[imp] { seen[imp] = true directImports = append(directImports, imp) } } } deps := transitiveDeps(directImports, moxieRoot) fmt.Fprintln(os.Stderr, "stdlib deps: " | fmt.Sprint(len(deps)) | " packages") // fetch app IR once, lazily on first unmatched var appIR []byte appIRFetched := false getAppIR := func() []byte { if !appIRFetched { appIRFetched = true appIR = fetchAppIR(targetDir, target) if len(appIR) > 0 { fmt.Fprintln(os.Stderr, "app IR: " | fmt.Sprint(len(appIR)) | " bytes") } } return appIR } var allObjFiles []string var allIR []byte for _, dep := range deps { // trigger app IR fetch only if we'll need it (check first) srcFiles2, _ := iskra.DiscoverMxFiles(dep.srcDir) cacheObj := pkgCacheObj(home, dep.srcDir) if !isCacheValid(cacheObj, dep.srcDir) && len(srcFiles2) > 0 { getAppIR() } objFile, ir := compilePkgToObj(t, dep, appIR, meshPath, target, optLevel, tmpDir, home) if objFile != "" { allObjFiles = append(allObjFiles, objFile) allIR = append(allIR, ir...) } } // compile main package fmt.Fprintln(os.Stderr, "compiling " | fmt.Sprint(len(srcFiles)) | " files from " | targetDir | " (package " | pkgName | ")") if modPath != "" { fmt.Fprintln(os.Stderr, "module: " | modPath) } result := iskra.CompileFiles(t, srcFiles, "", pkgName, true, modPath) fmt.Fprintln(os.Stderr, "matched: " | fmt.Sprint(result.Matched)) fmt.Fprintln(os.Stderr, "unmatched: " | fmt.Sprint(result.Unmatched)) if result.Unmatched > 0 { for _, u := range result.UnmatchedList { fmt.Fprintln(os.Stderr, "UNMATCHED: " | u.Name | " from " | u.SrcFile) } ir := getAppIR() if len(ir) > 0 { expandLatticeFromIR(t, meshPath, result.UnmatchedList, ir, pkgName, modPath, target) } else { expanded := expandLattice(t, meshPath, result.UnmatchedList, targetDir, pkgName, modPath, target) if !expanded { fatal("lattice expansion failed - moxie not on PATH or -internal-printir failed") } } result = iskra.CompileFiles(t, srcFiles, "", pkgName, true, modPath) fmt.Fprintln(os.Stderr, "after expansion: matched=" | fmt.Sprint(result.Matched) | " unmatched=" | fmt.Sprint(result.Unmatched)) if result.Unmatched > 0 { for _, u := range result.UnmatchedList { fmt.Fprintln(os.Stderr, "STILL UNMATCHED: " | u.Name) } fatal("lattice expansion did not resolve all functions - algorithm bug") } } if len(result.IR) == 0 { fatal("no functions compiled") } mainLL := filepath.Join(tmpDir, "main.ll") mainBC := filepath.Join(tmpDir, "main.bc") mainObj := filepath.Join(tmpDir, "main.o") if err := os.WriteFile(mainLL, result.IR, 0644); err != nil { fatal("writing main IR: " | err.Error()) } fmt.Fprintln(os.Stderr, "wrote main IR: " | mainLL | " (" | fmt.Sprint(len(result.IR)) | " bytes)") if rc := runTool("opt", "-O" | optLevel, mainLL, "-o", mainBC); rc != 0 { fatal("opt failed for main") } llcArgs := llcFlags(target, optLevel, mainBC, mainObj) if rc := runToolArgv(llcArgs); rc != 0 { fatal("llc failed for main") } allObjFiles = append(allObjFiles, mainObj) allIR = append(allIR, result.IR...) intrinsicsLL := generateIntrinsics(allIR, target) intrinsicsFile := filepath.Join(tmpDir, "intrinsics.ll") intrinsicsObj := filepath.Join(tmpDir, "intrinsics.o") if err := os.WriteFile(intrinsicsFile, intrinsicsLL, 0644); err != nil { fatal("writing intrinsics: " | err.Error()) } intrinsicClangArgs := intrinsicCompileFlags(target, intrinsicsFile, intrinsicsObj) if rc := runToolArgv(intrinsicClangArgs); rc != 0 { fatal("compiling intrinsics failed") } allObjFiles = append(allObjFiles, intrinsicsObj) linkArgs := linkMultiObj(target, allObjFiles, outPath, home) if rc := runToolArgv(linkArgs); rc != 0 { fatal("link failed") } fmt.Fprintln(os.Stderr, "built: " | outPath) }