package main import ( "os" "runtime" "unsafe" "git.smesh.lol/moxie/pkg/mxutil" . "git.smesh.lol/moxie/pkg/types" ) // buildState holds all build-configuration and persistent state that // outlives individual package compilations. Allocated once at init and // never reassigned; fields are mutated through the pointer. type buildState struct { selfHash string globalModPrefix string globalModDir string globalReplaces map[string]string buildJobs int32 quietDiscover bool wasmRtPkgs map[string]bool builtinOnly []string pkgKeyMemo map[string]string tlsProbeVar int32 } var bst *buildState func initBuildState() { if bst == nil { bst = &buildState{} } } //export mxc_run func cRun(argv unsafe.Pointer, argc int32) (n int32) //export mxc_run_async func cRunAsync(argv unsafe.Pointer, argc int32) (n int32) //export mxc_fork func cFork() (n int32) //export mxc_wait_any func cWaitAny() (n int32) //export mxc_getenv func cGetenv(name unsafe.Pointer, nameLen int32, buf unsafe.Pointer, bufCap int32) (n int32) //export mxc_listdir func cListdir(dir unsafe.Pointer, dirLen int32, buf unsafe.Pointer, bufCap int32) (n int32) //export mxc_writefile func cWritefile(path unsafe.Pointer, pathLen int32, data unsafe.Pointer, dataLen int32, mode uint32) (n int32) //export mxc_mkdir func cMkdir(path unsafe.Pointer, pathLen int32, mode uint32) (n int32) // Ring buffer FFI //export mxc_ring_create func cRingCreate(dataSize int32) (p unsafe.Pointer) //export mxc_ring_destroy func cRingDestroy(ring unsafe.Pointer) //export mxc_ring_close func cRingClose(ring unsafe.Pointer) //export mxc_ring_closed func cRingClosed(ring unsafe.Pointer) (n int32) //export mxc_ring_send func cRingSend(ring unsafe.Pointer, data unsafe.Pointer, dlen int32) (n int32) //export mxc_ring_peek_len func cRingPeekLen(ring unsafe.Pointer) (n int32) //export mxc_ring_recv func cRingRecv(ring unsafe.Pointer, buf unsafe.Pointer, bufcap int32) (n int32) // Thread FFI //export mxc_spawn_thread func cSpawnThread(fn unsafe.Pointer, arg unsafe.Pointer, handleOut unsafe.Pointer, statusOut unsafe.Pointer) (n int32) //export mxc_thread_alive func cThreadAlive(statusPtr unsafe.Pointer) (n int32) //export mxc_thread_join func cThreadJoin(handlePtr unsafe.Pointer, statusPtr unsafe.Pointer) //export mxc_usleep func cUsleep(us int32) //export mxc_spawn_worker func cSpawnWorker(arg unsafe.Pointer, handleOut unsafe.Pointer, statusOut unsafe.Pointer) (n int32) //export mxc_detect_tls func cDetectTLS() (n int32) // TLS probe: the C side spawns a thread that calls mxc_tls_probe_write, // then the parent calls mxc_tls_probe_read. If the global is TLS, the // child's write is invisible to the parent. //export mxc_tls_probe_write func tlsProbeWrite() { bst.tlsProbeVar = 1 } //export mxc_tls_probe_read func tlsProbeRead() (n int32) { return bst.tlsProbeVar } func writeFile(path string, data []byte, mode uint32) { if len(path) == 0 { return } cWritefile(unsafe.Pointer(unsafe.SliceData([]byte(path))), int32(len(path)), unsafe.Pointer(unsafe.SliceData([]byte(data))), int32(len(data)), mode) } func mkdirAll(path string, mode uint32) { if len(path) == 0 { return } cMkdir(unsafe.Pointer(unsafe.SliceData([]byte(path))), int32(len(path)), mode) } func fatal(msg string) { mxutil.WriteStr(2, "mxc: " | msg | "\n") os.Exit(1) } func clangCompileToObj(clang, triple, bcFile string) (s string) { oFile := bcFile | ".o" rc := run([]string{clang, "--target=" | triple, "-c", bcFile, "-o", oFile}) if rc != 0 { fatal("clang -c failed for " | bcFile) } return oFile } func getenv(name string) (s string) { buf := []byte{:4096} n := cGetenv(unsafe.Pointer(unsafe.SliceData([]byte(name))), int32(len(name)), unsafe.Pointer(unsafe.SliceData(buf)), int32(len(buf))) if n <= 0 { return "" } return string(buf[:n]) } func listFiles(dir, ext string) (ss []string) { buf := []byte{:65536} n := cListdir(unsafe.Pointer(unsafe.SliceData([]byte(dir))), int32(len(dir)), unsafe.Pointer(unsafe.SliceData(buf)), int32(len(buf))) if n <= 0 { return nil } // Count null-separated entries matching ext. nc := int32(0) start := int32(0) for i := int32(0); i < n; i++ { if buf[i] == 0 { if mxutil.HasSuffix(string(buf[start:i]), ext) { nc++ } start = i + 1 } } if nc == 0 { return nil } result := []string{:0:nc} start = 0 for i := int32(0); i < n; i++ { if buf[i] == 0 { name := string(buf[start:i]) if mxutil.HasSuffix(name, ext) { push(result, name) } start = i + 1 } } return result } func run(args []string) (n int32) { argv := []unsafe.Pointer{:len(args) + 1} cstrs := [][]byte{:len(args)} for i, a := range args { b := []byte{:len(a) + 1} copy(b, a) b[len(a)] = 0 cstrs[i] = b argv[i] = unsafe.Pointer(unsafe.SliceData(b)) } return cRun(unsafe.Pointer(unsafe.SliceData(argv)), int32(len(args))) } // runAsync forks+execs args without waiting. Returns child pid, or -1 on // fork failure. The forked child copies the argv buffers, so parent-side // lifetimes end at return. func runAsync(args []string) (pid int32) { argv := []unsafe.Pointer{:len(args) + 1} cstrs := [][]byte{:len(args)} for i, a := range args { b := []byte{:len(a) + 1} copy(b, a) b[len(a)] = 0 cstrs[i] = b argv[i] = unsafe.Pointer(unsafe.SliceData(b)) } return cRunAsync(unsafe.Pointer(unsafe.SliceData(argv)), int32(len(args))) } // runPool runs cmds with at most jobs concurrent children. Returns false if // any command exits nonzero (remaining in-flight children are drained, no // new ones are dispatched). func runPool(cmds [][]string, jobs int32) (ok bool) { if jobs < 1 { jobs = 1 } next := int32(0) running := int32(0) failed := false for { for !failed && running < jobs && next < int32(len(cmds)) { pid := runAsync(cmds[next]) if pid < 0 { failed = true break } running++ next++ } if running == 0 { break } r := cWaitAny() if r == -1 { return false } running-- if r < 0 { failed = true } } return !failed } func parseInt32(s string) (n int32) { v := int32(0) for i := int32(0); i < int32(len(s)); i++ { if s[i] < '0' || s[i] > '9' { return 0 } v = v*10 + int32(s[i]-'0') } return v } func pathBase(path string) (s string) { for i := int32(len(path)) - 1; i >= 0; i-- { if path[i] == '/' { return path[i+1:] } } return path } func pathDir(path string) (s string) { for i := int32(len(path)) - 1; i >= 0; i-- { if path[i] == '/' { if i == 0 { return "/" } return path[:i] } } return "." } func splitSlash(s string) (ss []string) { var parts []string start := int32(0) for i := int32(0); i < int32(len(s)); i++ { if s[i] == '/' { push(parts, s[start:i]) start = i + 1 } } push(parts, s[start:]) return parts } func cleanPath(path string) (s string) { parts := splitSlash(path) var out []string for _, p := range parts { if p == "." || p == "" { continue } if p == ".." && len(out) > 0 && out[len(out)-1] != ".." { out = out[:len(out)-1] } else { push(out, p) } } if len(out) == 0 { return "." } result := out[0] for i := 1; i < len(out); i++ { result = result | "/" | out[i] } if len(path) > 0 && path[0] == '/' { result = "/" | result } return result } func getMoxiePath() (s string) { p := getenv("MOXIEPATH") if p != "" { return p } home := getenv("HOME") if home == "" { return "/tmp/moxie" } return mxutil.JoinPath(home, "moxie") } func mxcVersion() (s string) { return "1.9.10" } func compilerHash() (s string) { if bst.selfHash != "" { return bst.selfHash } bin, ok := mxutil.ReadFile("/proc/self/exe") if !ok { bst.selfHash = "unknown" return bst.selfHash } bst.selfHash = fnvHash(bin) return bst.selfHash } func moxieCacheDir() (s string) { return mxutil.JoinPath(mxutil.JoinPath(getMoxiePath(), "cache"), "mxc-" | mxcVersion() | "-" | compilerHash()) } func parseReplaceLine(line string, baseDir string) (from string, to string, ok bool) { arrow := int32(-1) for k := int32(0); k < int32(len(line))-1; k++ { if line[k] == '=' && line[k+1] == '>' { arrow = k break } } if arrow < 0 { return "", "", false } from := mxutil.TrimSpace(line[:arrow]) to := mxutil.TrimSpace(line[arrow+2:]) for fi := int32(0); fi < int32(len(from)); fi++ { if from[fi] == ' ' { from = from[:fi] break } } if mxutil.HasPrefix(to, "./") || mxutil.HasPrefix(to, "../") || mxutil.HasPrefix(to, "/") { to = cleanPath(mxutil.JoinPath(baseDir, to)) } return from, to, true } func findModuleRoot(startDir string) (modName string, dir string) { dir = startDir for { data, ok := mxutil.ReadFile(mxutil.JoinPath(dir, "moxie.mod")) if ok { modName = "" bst.globalReplaces = map[string]string{} lines := mxutil.SplitLines(string(data)) inReplace := false for _, line := range lines { line = mxutil.TrimSpace(line) if mxutil.HasPrefix(line, "module ") { modName = mxutil.TrimSpace(line[7:]) continue } if line == "replace (" { inReplace = true continue } if inReplace { if line == ")" { inReplace = false continue } from, to, ok2 := parseReplaceLine(line, dir) if ok2 { bst.globalReplaces[from] = to } continue } if mxutil.HasPrefix(line, "replace ") { from, to, ok2 := parseReplaceLine(line[8:], dir) if ok2 { bst.globalReplaces[from] = to } } } if modName != "" { return modName, dir } } parent := pathDir(dir) if parent == dir || parent == "" || parent == "." { break } dir = parent } return "", "" } func mergeModReplaces(dir string) { data, ok := mxutil.ReadFile(mxutil.JoinPath(dir, "moxie.mod")) if !ok { return } if bst.globalReplaces == nil { bst.globalReplaces = map[string]string{} } inReplace := false for _, line := range mxutil.SplitLines(string(data)) { line = mxutil.TrimSpace(line) if line == "replace (" { inReplace = true continue } if inReplace { if line == ")" { inReplace = false continue } from, to, ok2 := parseReplaceLine(line, dir) if ok2 { if _, exists := bst.globalReplaces[from]; !exists { bst.globalReplaces[from] = to } } continue } if mxutil.HasPrefix(line, "replace ") { from, to, ok2 := parseReplaceLine(line[8:], dir) if ok2 { if _, exists := bst.globalReplaces[from]; !exists { bst.globalReplaces[from] = to } } } } } type modRequire struct { path string version string } func parseModRequires(dir string) (modName string, reqs []modRequire, repls map[string]string) { data, ok := mxutil.ReadFile(mxutil.JoinPath(dir, "moxie.mod")) if !ok { return "", nil, nil } modName = "" repls = map[string]string{} inRequire := false inReplace := false for _, line := range mxutil.SplitLines(string(data)) { line = mxutil.TrimSpace(line) if mxutil.HasPrefix(line, "module ") { modName = mxutil.TrimSpace(line[7:]) continue } if line == "require (" { inRequire = true continue } if inRequire { if line == ")" { inRequire = false continue } parts := splitBySpace(line) if len(parts) >= 2 { push(reqs, modRequire{path: parts[0], version: parts[1]}) } else if len(parts) == 1 { push(reqs, modRequire{path: parts[0], version: ""}) } continue } if mxutil.HasPrefix(line, "require ") { parts := splitBySpace(line[8:]) if len(parts) >= 2 { push(reqs, modRequire{path: parts[0], version: parts[1]}) } else if len(parts) == 1 { push(reqs, modRequire{path: parts[0], version: ""}) } continue } if line == "replace (" { inReplace = true continue } if inReplace { if line == ")" { inReplace = false continue } from, to, ok2 := parseReplaceLine(line, dir) if ok2 { repls[from] = to } continue } if mxutil.HasPrefix(line, "replace ") { from, to, ok2 := parseReplaceLine(line[8:], dir) if ok2 { repls[from] = to } } } return modName, reqs, repls } func isRemotePath(path string) (ok bool) { slash := int32(-1) for i := int32(0); i < int32(len(path)); i++ { if path[i] == '/' { slash = i break } } if slash < 0 { return false } host := path[:slash] for i := int32(0); i < int32(len(host)); i++ { if host[i] == '.' { return true } } return false } func repoLockPath(mpath, repoPath string) (s string) { return mxutil.JoinPath(mpath, repoPath | ".lock") } func acquireRepoLock(mpath, repoPath string) (ok bool) { lp := repoLockPath(mpath, repoPath) _, locked := mxutil.ReadFile(lp) if locked { return false } mkdirAll(pathDir(lp), 0755) writeFile(lp, []byte("1"), 0644) return true } func releaseRepoLock(mpath, repoPath string) { os.Remove(repoLockPath(mpath, repoPath)) } func autoFetchRepo(pkgPath, version string) (s string) { mpath := getMoxiePath() repoPath := pkgPath for { d := mxutil.JoinPath(mpath, repoPath) gh := mxutil.JoinPath(d, ".git/HEAD") _, ex := mxutil.ReadFile(gh) if ex { break } parent := pathDir(repoPath) if parent == repoPath || parent == "." || parent == "" { break } parentDest := mxutil.JoinPath(mpath, parent) parentGit := mxutil.JoinPath(parentDest, ".git/HEAD") _, parentExists := mxutil.ReadFile(parentGit) if parentExists { repoPath = parent break } repoPath = parent } if !acquireRepoLock(mpath, repoPath) { mxutil.WriteStr(2, " waiting for lock: " | repoPath | "\n") fatal("repo locked: " | repoPath) } dest := mxutil.JoinPath(mpath, repoPath) gitHead := mxutil.JoinPath(dest, ".git/HEAD") _, exists := mxutil.ReadFile(gitHead) if !exists { mkdirAll(pathDir(dest), 0755) url := "https://" | repoPath mxutil.WriteStr(2, " auto-fetch " | repoPath | "\n") if run([]string{"git", "clone", url, dest}) != 0 { releaseRepoLock(mpath, repoPath) return "" } } if version != "" && version != "v0.0.0" { if run([]string{"git", "-C", dest, "checkout", version}) != 0 { run([]string{"git", "-C", dest, "fetch", "--tags"}) run([]string{"git", "-C", dest, "pull"}) run([]string{"git", "-C", dest, "checkout", version}) } } releaseRepoLock(mpath, repoPath) return mxutil.JoinPath(mpath, pkgPath) } func gitURLToModPath(url string) (s string) { u := url // strip scheme for _, scheme := range []string{"ssh://", "https://", "http://", "git://"} { if mxutil.HasPrefix(u, scheme) { u = u[len(scheme):] break } } // strip user@ for i := int32(0); i < int32(len(u)); i++ { if u[i] == '@' { u = u[i+1:] break } if u[i] == '/' { break } } // host:port/path or host:path (scp-style) or host/path // extract host, skip port if numeric, keep path host := "" rest := "" for i := int32(0); i < int32(len(u)); i++ { if u[i] == ':' { host = u[:i] after := u[i+1:] // skip port (digits until /) j := int32(0) for j < int32(len(after)) && after[j] >= '0' && after[j] <= '9' { j++ } if j > 0 && j < int32(len(after)) && after[j] == '/' { rest = after[j+1:] } else if j > 0 && j == int32(len(after)) { rest = "" } else { rest = after } break } if u[i] == '/' { host = u[:i] rest = u[i+1:] break } } if host == "" { return "" } // strip ~/ prefix from path if mxutil.HasPrefix(rest, "~/") { rest = rest[2:] } // strip .git suffix if mxutil.HasSuffix(rest, ".git") { rest = rest[:len(rest)-4] } if rest == "" { return "" } return host | "/" | rest } func currentBranch(dest string) (s string) { data, ok := mxutil.ReadFile(mxutil.JoinPath(dest, ".git/HEAD")) if !ok { return "" } line := mxutil.TrimSpace(string(data)) if !mxutil.HasPrefix(line, "ref: refs/heads/") { return "" } return line[len("ref: refs/heads/"):] } func cmdFetch(url string) { mpath := getMoxiePath() mkdirAll(mpath, 0755) rootDir := "." if url != "" { modPath := gitURLToModPath(url) if modPath == "" { fatal("cannot derive module path from URL: " | url) } dest := mxutil.JoinPath(mpath, modPath) _, exists := mxutil.ReadFile(mxutil.JoinPath(dest, ".git/HEAD")) if exists { mxutil.WriteStr(2, modPath | ": fetch\n") run([]string{"git", "-C", dest, "remote", "set-url", "origin", url}) run([]string{"git", "-C", dest, "fetch", "origin"}) // older clones lack the origin/HEAD symbolic ref; resolve it // from the remote before resetting to it run([]string{"git", "-C", dest, "remote", "set-head", "origin", "-a"}) if run([]string{"git", "-C", dest, "reset", "--hard", "origin/HEAD"}) != 0 { // server advertises no HEAD (default branch unset); // fall back to the upstream of the current branch br := currentBranch(dest) if br != "" { run([]string{"git", "-C", dest, "reset", "--hard", "origin/" | br}) } } } else { mxutil.WriteStr(2, modPath | ": clone\n") mkdirAll(pathDir(dest), 0755) if run([]string{"git", "clone", url, dest}) != 0 { fatal("clone failed: " | url) } } rootDir = dest } modName, reqs, repls := parseModRequires(rootDir) if modName == "" { fatal("no moxie.mod in " | rootDir) } if url == "" { mxutil.WriteStr(2, "module: " | modName | "\n") } queue := []modRequire{:0:len(reqs)} for _, r := range reqs { if _, ok := repls[r.path]; ok { mxutil.WriteStr(2, " skip (replaced): " | r.path | "\n") continue } push(queue, r) } seen := map[string]bool{} for len(queue) > 0 { req := queue[0] queue = queue[1:] if seen[req.path] { continue } seen[req.path] = true if !acquireRepoLock(mpath, req.path) { mxutil.WriteStr(2, " locked: " | req.path | " (another build in progress)\n") fatal("repo locked: " | req.path) } dest := mxutil.JoinPath(mpath, req.path) gitHead := mxutil.JoinPath(dest, ".git/HEAD") _, exists := mxutil.ReadFile(gitHead) if !exists { mkdirAll(pathDir(dest), 0755) cloneURL := "https://" | req.path mxutil.WriteStr(2, " clone " | req.path | "\n") if run([]string{"git", "clone", cloneURL, dest}) != 0 { releaseRepoLock(mpath, req.path) mxutil.WriteStr(2, " FAIL: clone " | req.path | "\n") continue } } if req.version != "" && req.version != "v0.0.0" { if run([]string{"git", "-C", dest, "checkout", req.version}) != 0 { mxutil.WriteStr(2, " tag " | req.version | " not found, pulling\n") run([]string{"git", "-C", dest, "pull"}) if run([]string{"git", "-C", dest, "checkout", req.version}) != 0 { mxutil.WriteStr(2, " WARN: " | req.version | " unavailable for " | req.path | "\n") } } else { mxutil.WriteStr(2, " " | req.path | " @ " | req.version | "\n") } } else if exists { mxutil.WriteStr(2, " pull " | req.path | "\n") run([]string{"git", "-C", dest, "pull"}) } else { mxutil.WriteStr(2, " fetched " | req.path | "\n") } releaseRepoLock(mpath, req.path) _, subReqs, _ := parseModRequires(dest) for _, sr := range subReqs { if !seen[sr.path] { push(queue, sr) } } } mxutil.WriteStr(2, "fetch done\n") } func joinStrings(ss []string, sep string) (s string) { if len(ss) == 0 { return "" } n := int32(0) for _, v := range ss { n += int32(len(v)) } n += int32(len(sep)) * (int32(len(ss)) - 1) buf := []byte{:0:n} for i, v := range ss { if i > 0 { buf = buf | sep } buf = buf | v } return string(buf) } func appendUniq(ss []string, s string) (ss2 []string) { for _, x := range ss { if x == s { return ss } } push(ss, s) return ss } func fnvHash(data []byte) (s string) { h := uint32(2166136261) for i := int32(0); i < int32(len(data)); i++ { h ^= uint32(data[i]) h *= uint32(16777619) } hex := "0123456789abcdef" buf := []byte{:8} for i := int32(7); i >= 0; i-- { buf[i] = hex[h&0xf] h >>= 4 } return string(buf) } func pkgCacheDir() (s string) { if mxutil.TargetArch == "wasm" { return mxutil.JoinPath(moxieCacheDir(), "pkg-wasm") } return mxutil.JoinPath(moxieCacheDir(), "pkg") } func pkgCacheKey(dir string, files []string) (s string) { var sorted []string for _, f := range files { push(sorted, f) } for i := int32(1); i < int32(len(sorted)); i++ { for j := i; j > 0 && sorted[j] < sorted[j-1]; j-- { sorted[j], sorted[j-1] = sorted[j-1], sorted[j] } } var all []byte for _, f := range sorted { p := mxutil.JoinPath(dir, f) data, ok := mxutil.ReadFile(p) if !ok { continue } all = all | data push(all, 0) } return fnvHash(all) } // pkgKeyMemo caches each package's transitive hash by import path. Filled // in dependency order (resolver emits deps before dependents), so a // dependent's key always sees its deps' final hashes. Without dep hashes in // the key, a dependency's layout change would leave a dependent's cached .bc // reusable with a stale ABI - silent struct-offset corruption at link time. func pkgHash(p *pkgInfo) (s string) { if bst.pkgKeyMemo == nil { bst.pkgKeyMemo = map[string]string{} } if h, ok := bst.pkgKeyMemo[p.path]; ok { return h } var sorted []string for _, imp := range p.imports { push(sorted, imp) } for i := int32(1); i < int32(len(sorted)); i++ { for j := i; j > 0 && sorted[j] < sorted[j-1]; j-- { sorted[j], sorted[j-1] = sorted[j-1], sorted[j] } } acc := []byte(pkgCacheKey(p.dir, p.files)) for _, imp := range sorted { if dh, ok := bst.pkgKeyMemo[imp]; ok { push(acc, '|') acc = acc | dh } } hv := fnvHash(acc) bst.pkgKeyMemo[p.path] = hv return hv } func versionedCacheName(hash, version, ext string) (s string) { if version != "" { return hash | "_" | version | ext } return hash | ext } func cachedBcPath(cacheBase, pkgPath, hash, version string) (s string) { return mxutil.JoinPath(mxutil.JoinPath(cacheBase, safeName(pkgPath)), versionedCacheName(hash, version, ".bc")) } func isStdlib(path string) (ok bool) { return !mxutil.HasPrefix(path, "/") && !mxutil.HasPrefix(path, "./") && !mxutil.HasPrefix(path, "../") } func isBuiltinOnly(path string) (ok bool) { for _, b := range bst.builtinOnly { if path == b { return true } } return false } func loadSysroot(path string) { data, ok := mxutil.ReadFile(path) if !ok { return } for _, line := range mxutil.SplitLines(string(data)) { line = mxutil.TrimSpace(line) if len(line) == 0 || line[0] == '#' { continue } eq := int32(-1) for i := int32(0); i < int32(len(line)); i++ { if line[i] == '=' { eq = i break } } if eq < 0 { continue } key := line[:eq] val := line[eq+1:] os.Setenv(key, val) } } func splitBySpace(s string) (ss []string) { var result []string start := int32(0) inWord := false for i := int32(0); i < int32(len(s)); i++ { if s[i] == ' ' || s[i] == '\t' { if inWord { push(result, s[start:i]) inWord = false } } else { if !inWord { start = i inWord = true } } } if inWord { push(result, s[start:]) } return result } func safeName(pkg string) (s string) { b := []byte{:len(pkg)} for i := int32(0); i < int32(len(pkg)); i++ { if pkg[i] == '/' { b[i] = '_' } else { b[i] = pkg[i] } } return string(b) } func extractPkgName(data []byte) (s string) { for i := int32(0); i < int32(len(data)); i++ { if i+8 < int32(len(data)) && string(data[i:i+8]) == "package " { j := i + 8 for j < int32(len(data)) && data[j] != '\n' && data[j] != '\r' && data[j] != ' ' { j++ } return string(data[i+8 : j]) } if data[i] == '\n' { continue } if data[i] == '/' && i+1 < int32(len(data)) && data[i+1] == '/' { for i < int32(len(data)) && data[i] != '\n' { i++ } continue } // Block comment: skip to closing */ (fmt/doc.mx has its package // clause below a /* ... */ doc comment). if data[i] == '/' && i+1 < int32(len(data)) && data[i+1] == '*' { i += 2 for i+1 < int32(len(data)) && !(data[i] == '*' && data[i+1] == '/') { i++ } i++ // lands on '/', loop increment moves past it continue } if data[i] == ' ' || data[i] == '\t' || data[i] == '\r' { continue } break } return "main" } func extractImports(data []byte) (ss []string) { imports := []string{:0:32} off := int32(0) for off < int32(len(data)) { ls := off for off < int32(len(data)) && data[off] != '\n' { off++ } le := off if off < int32(len(data)) { off++ } // Trim for ls < le && (data[ls] == ' ' || data[ls] == '\t') { ls++ } for le > ls && (data[le-1] == ' ' || data[le-1] == '\t' || data[le-1] == '\r') { le-- } line := string(data[ls:le]) if line == "import (" { // Scan block for off < int32(len(data)) { bls := off for off < int32(len(data)) && data[off] != '\n' { off++ } ble := off if off < int32(len(data)) { off++ } for bls < ble && (data[bls] == ' ' || data[bls] == '\t') { bls++ } for ble > bls && (data[ble-1] == ' ' || data[ble-1] == '\t' || data[ble-1] == '\r') { ble-- } bt := string(data[bls:ble]) if bt == ")" { break } imp := mxutil.ExtractQuoted(bt) if imp != "" { imports = appendUniq(imports, imp) } } continue } if mxutil.HasPrefix(line, "import ") { imp := mxutil.ExtractQuoted(line[7:]) if imp != "" { imports = appendUniq(imports, imp) } } } return imports } type srcSpan struct { file string outStart int32 srcLine int32 } func concatLineToFile(concatLine int32) (file string, srcLine int32) { if len(cctx.concatSourceMap) == 0 { return "", concatLine } best := int32(0) for i, s := range cctx.concatSourceMap { if s.outStart <= concatLine { best = int32(i) } } sp := cctx.concatSourceMap[best] return sp.file, sp.srcLine + (concatLine - sp.outStart) } // scanFileRegions scans file bytes to find the body start offset and // extract import specs. No SplitLines, no per-line copies. Returns // bodyStart (byte offset of first non-package/import line) and the // source line number of that offset (1-based). func scanFileRegions(data []byte, imports map[string]bool) (bodyStart int32, bodyLine int32, fimps []string) { off := int32(0) lineNum := int32(1) bodyStart = -1 for off < int32(len(data)) { // Find line extent lineStart := off for off < int32(len(data)) && data[off] != '\n' { off++ } lineEnd := off if off < int32(len(data)) { off++ // skip \n } // Trim whitespace for classification ls := lineStart for ls < lineEnd && (data[ls] == ' ' || data[ls] == '\t') { ls++ } le := lineEnd for le > ls && (data[le-1] == ' ' || data[le-1] == '\t' || data[le-1] == '\r') { le-- } trimmed := data[ls:le] if len(trimmed) >= 8 && string(trimmed[:8]) == "package " { lineNum++ continue } if string(trimmed) == "import (" { lineNum++ // Scan import block for off < int32(len(data)) { bls := off for off < int32(len(data)) && data[off] != '\n' { off++ } ble := off if off < int32(len(data)) { off++ } // Trim for bls < ble && (data[bls] == ' ' || data[bls] == '\t') { bls++ } for ble > bls && (data[ble-1] == ' ' || data[ble-1] == '\t' || data[ble-1] == '\r') { ble-- } bt := data[bls:ble] lineNum++ if string(bt) == ")" { break } if len(bt) > 0 { spec := string(bt) imports[spec] = true push(fimps, spec) } } continue } if len(trimmed) >= 7 && string(trimmed[:7]) == "import " && (len(trimmed) < 8 || trimmed[7] != '(') { spec := string(trimmed[7:]) imports[spec] = true push(fimps, spec) lineNum++ continue } // First body line if bodyStart < 0 { bodyStart = lineStart bodyLine = lineNum } lineNum++ } if bodyStart < 0 { bodyStart = int32(len(data)) bodyLine = lineNum } return bodyStart, bodyLine, fimps } func concatSources(dir string, files []string) (buf []byte) { imports := map[string]bool{} pkgName := "" // First pass: scan each file for imports and body boundaries. // Store body as a byte slice reference into the file data. type fileBody struct { name string data []byte // full file data (freed after copy into output) bodyStart int32 bodyLine int32 fimps []string } var fb []fileBody for _, f := range files { data, ok := mxutil.ReadFile(mxutil.JoinPath(dir, f)) if !ok { continue } if pkgName == "" { pkgName = extractPkgName(data) } bs, bl, fimps := scanFileRegions(data, imports) push(fb, fileBody{name: f, data: data, bodyStart: bs, bodyLine: bl, fimps: fimps}) } // Estimate total size for pre-allocation totalSize := int32(64) // header for _, f := range fb { totalSize += int32(len(f.data)) - f.bodyStart + 1 } for imp := range imports { totalSize += int32(len(imp)) + 2 } var out []byte out = out | ("package " | pkgName | "\n") if len(imports) > 0 { var impKeys []string for imp := range imports { push(impKeys, imp) } for i := int32(1); i < int32(len(impKeys)); i++ { for j := i; j > 0 && impKeys[j] < impKeys[j-1]; j-- { impKeys[j], impKeys[j-1] = impKeys[j-1], impKeys[j] } } out = out | "import (\n" for _, imp := range impKeys { push(out, '\t') out = out | imp push(out, '\n') } out = out | ")\n" } cctx.concatSourceMap = nil mxutil.ResetConcatImportSegs() outLine := int32(1) for j := int32(0); j < int32(len(out)); j++ { if out[j] == '\n' { outLine++ } } var bodyChunks [][]byte push(bodyChunks, out) for i := range fb { body := fb[i].data[fb[i].bodyStart:] push(cctx.concatSourceMap, srcSpan{ file: fb[i].name, outStart: outLine, srcLine: fb[i].bodyLine, }) mxutil.AppendConcatImportSeg(mxutil.ImportSeg{ OutStart: outLine, Specs: fb[i].fimps, }) for j := int32(0); j < int32(len(body)); j++ { if body[j] == '\n' { outLine++ } } outLine++ push(bodyChunks, body) push(bodyChunks, []byte("\n")) } return joinChunks(bodyChunks) } func fileContainsPart(base string, part string) (ok bool) { plen := len(part) for i := 0; i <= len(base)-plen; i++ { if base[i:i+plen] == part { before := i == 0 || base[i-1] == '_' after := i+plen == len(base) || base[i+plen] == '_' if before && after { return true } } } return false } func shouldSkipFile(name string) (ok bool) { if mxutil.HasSuffix(name, "_test.mx") { return true } base := name if mxutil.HasSuffix(base, ".mx") { base = base[:len(base)-3] } if mxutil.TargetArch == "wasm" { for _, a := range []string{"amd64", "arm64", "386"} { if fileContainsPart(base, a) { return true } } for _, s := range []string{"_linux", "_unix", "_darwin", "_posix"} { if mxutil.HasSuffix(base, s) { return true } } } else { for _, a := range []string{"arm64"} { if fileContainsPart(base, a) { return true } } if mxutil.HasSuffix(base, "_wasm") { return true } } return false } func isTagChar(c byte) (ok bool) { return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_' || c == '.' } func constraintContainsPositive(line string, word string) (ok bool) { wl := int32(len(word)) for i := int32(0); i <= int32(len(line))-wl; i++ { if string(line[i:i+wl]) == word { if i > 0 && line[i-1] == '!' { continue } after := i + wl if after < int32(len(line)) { if isTagChar(line[after]) { continue } } if i > 0 { if isTagChar(line[i-1]) { continue } } // check if inside a negated group !(...) if isInsideNegatedGroup(line, i) { continue } return true } } return false } func isInsideNegatedGroup(line string, pos int32) (result bool) { // scan backwards from pos for !(, counting parens for j := pos - 1; j >= 1; j-- { if line[j] == '(' && line[j-1] == '!' { // found !(, now check the matching ) is after pos depth := int32(1) k := j + 1 for k < int32(len(line)) && depth > 0 { if line[k] == '(' { depth++ } if line[k] == ')' { depth-- } k++ } if k-1 > pos { // closing ) is after our position return true } } } return false } func constraintContainsNegated(line string, word string) (ok bool) { wl := int32(len(word)) // check direct negation: !word for i := int32(1); i <= int32(len(line))-wl; i++ { if string(line[i:i+wl]) == word && line[i-1] == '!' { after := i + wl if after < int32(len(line)) { if isTagChar(line[after]) { continue } } return true } } // check parenthesized negation: !(... word ...) for i := int32(0); i+1 < int32(len(line)); i++ { if line[i] == '!' && line[i+1] == '(' { depth := int32(1) j := i + 2 for j < int32(len(line)) && depth > 0 { if line[j] == '(' { depth++ } if line[j] == ')' { depth-- } j++ } group := string(line[i+2 : j-1]) if constraintContainsPositive(group, word) { return true } } } return false } func hasBuildConstraint(data []byte, forbidden []string) (ok bool) { for i := int32(0); i < int32(len(data)) && i < 512; i++ { if data[i] == 'p' { return false } skip := int32(0) if i+9 < int32(len(data)) && string(data[i:i+9]) == "//:build " { skip = 9 } if skip > 0 { var buf []byte for j := i + skip; j < int32(len(data)) && data[j] != '\n'; j++ { push(buf, data[j]) } line := string(buf) if mxutil.TargetArch == "wasm" { // Tags true for all wasm: wasm, moxie.wasm, go1.23, !linux, !baremetal // js tag only true when mxutil.TargetOS == "js" (js/wasm target) if constraintContainsPositive(line, "moxie.wasm") || constraintContainsPositive(line, "wasm") || constraintContainsPositive(line, "go1.23") { // Only reject js requirement when it's conjunctive (js && wasm) // not disjunctive (js || wasm) if constraintContainsPositive(line, "js") && mxutil.TargetOS != "js" { // Check if js and wasm are in a conjunction isConjunction := false for ci := int32(0); ci < int32(len(line))-1; ci++ { if line[ci] == '&' && line[ci+1] == '&' { isConjunction = true break } } if isConjunction { return true } } return false } if mxutil.TargetOS == "js" && constraintContainsPositive(line, "js") { return false } if constraintContainsNegated(line, "js") || constraintContainsNegated(line, "wasm") || constraintContainsNegated(line, "moxie.wasm") || constraintContainsNegated(line, "go1.23") { return true } // Architecture tags: amd64, arm64, 386 are false on wasm. for _, arch := range []string{"amd64", "arm64", "386"} { if constraintContainsPositive(line, arch) { return true } } // !linux is true for wasm - include files that need it if constraintContainsNegated(line, "linux") { return false } // linux-only constraints don't apply to wasm if constraintContainsPositive(line, "linux") { return true } } for _, f := range forbidden { if line == f || mxutil.HasPrefix(line, f | " ") || mxutil.HasPrefix(line, f | "\t") || mxutil.HasPrefix(line, f | "(") { return true } } return false } if data[i] == '\n' { continue } if data[i] == '/' && i+1 < int32(len(data)) && data[i+1] == '/' { for i < int32(len(data)) && data[i] != '\n' { i++ } continue } } return false } func hasImport(data []byte, pkg string) (ok bool) { target := "\"" | pkg | "\"" for i := int32(0); i <= int32(len(data))-int32(len(target)); i++ { if string(data[i:i+int32(len(target))]) == target { return true } } return false } // quietDiscover suppresses per-file discovery chatter. Set by build-pkg // workers, which re-run discovery the orchestrator already printed. // buildJobs is the max number of concurrent child processes (workers and // clang) used by build/test. Set by the -j flag; 1 forces the serial path. // Packages baked into sysroot/wasm/runtime.bc (compiled by the old // TinyGo-based compiler). Stage4 type-checks these but does not compile // them - the prebuilt versions are used instead. This avoids ABI // mismatches between old-compiler and stage4-compiled code (different // context-pointer conventions). type pkgInfo struct { path string name string dir string files []string cfiles []string bcfiles []string imports []string version string } func discoverPkg(pkgPath, root string) (p *pkgInfo) { var dir string if pkgPath == "." || pkgPath == ".." || mxutil.HasPrefix(pkgPath, "/") || mxutil.HasPrefix(pkgPath, "./") || mxutil.HasPrefix(pkgPath, "../") { dir = pkgPath } else if bst.globalReplaces != nil { if rdir, ok := bst.globalReplaces[pkgPath]; ok { dir = rdir } if dir == "" { for from, to := range bst.globalReplaces { if mxutil.HasPrefix(pkgPath, from|"/") { rel := pkgPath[len(from)+1:] dir = mxutil.JoinPath(to, rel) break } } } } if dir == "" && bst.globalModPrefix != "" && pkgPath == bst.globalModPrefix { dir = bst.globalModDir } if dir == "" && bst.globalModPrefix != "" && mxutil.HasPrefix(pkgPath, bst.globalModPrefix|"/") { rel := pkgPath[len(bst.globalModPrefix)+1:] dir = mxutil.JoinPath(bst.globalModDir, rel) } if dir == "" { candidate := mxutil.JoinPath(getMoxiePath(), pkgPath) mxF := listFiles(candidate, ".mx") if len(mxF) > 0 { dir = candidate } } if dir == "" && isRemotePath(pkgPath) { ver := "" if bst.globalModDir != "" { _, reqs, _ := parseModRequires(bst.globalModDir) for _, r := range reqs { if r.path == pkgPath { ver = r.version break } } } fetched := autoFetchRepo(pkgPath, ver) if fetched != "" { mxF := listFiles(fetched, ".mx") if len(mxF) > 0 { dir = fetched } } } if dir != "" && dir != pkgPath { mergeModReplaces(dir) } if dir == "" { dir = mxutil.JoinPath(root, mxutil.JoinPath("src", pkgPath)) mxf := listFiles(dir, ".mx") if len(mxf) == 0 { dir = mxutil.JoinPath(root, mxutil.JoinPath("src/vendor", pkgPath)) } } mxFiles := listFiles(dir, ".mx") bcFiles := listFiles(dir, ".bc") allMx := mxFiles var forbidden []string if mxutil.TargetArch == "wasm" { forbidden = []string{"none", "linux", "unix", "darwin", "amd64", "arm64", "ignore", "compiler_bootstrap", "scheduler.tasks", "scheduler.cores", "scheduler.asyncify", "faketime", "baremetal", "nintendoswitch", "boringcrypto", "cgo", "runtime_memhash_leveldb", "runtime_memhash_tsip", "runtime_asserts", "moxie.stringer"} } else { forbidden = []string{"none", "wasm", "js", "ignore", "compiler_bootstrap", "scheduler.tasks", "scheduler.cores", "scheduler.asyncify", "faketime", "baremetal", "nintendoswitch", "boringcrypto", "gc"} } cFilesAll := listFiles(dir, ".c") cFiles := []string{:0:len(cFilesAll)} for _, cf := range cFilesAll { if mxutil.HasSuffix(cf, "_test.c") { continue } if shouldSkipFile(cf) { continue } cdata, cok := mxutil.ReadFile(mxutil.JoinPath(dir, cf)) if cok && hasBuildConstraint(cdata, forbidden) { continue } push(cFiles, cf) } files := []string{:0:len(allMx)} for _, f := range allMx { if shouldSkipFile(f) { continue } data, ok := mxutil.ReadFile(mxutil.JoinPath(dir, f)) if !ok { continue } bc := hasBuildConstraint(data, forbidden) if bc { if !bst.quietDiscover { mxutil.WriteStr(2, " skip-constraint: " | f | "\n") } continue } fileImps := extractImports(data) skipFile := "" for _, imp := range fileImps { if imp == "iter" || imp == "weak" || imp == "crypto/internal/fips140cache" { skipFile = imp break } } if skipFile != "" { if !bst.quietDiscover { mxutil.WriteStr(2, " skip-import(" | skipFile | "): " | f | "\n") } continue } if !bst.quietDiscover { mxutil.WriteStr(2, " include: " | f | "\n") } push(files, f) } if len(files) == 0 { return nil } allImports := []string{:0:len(files) * 4} pkgName := "" for _, f := range files { data, ok := mxutil.ReadFile(mxutil.JoinPath(dir, f)) if !ok { continue } if pkgName == "" { pkgName = extractPkgName(data) } for _, imp := range extractImports(data) { allImports = appendUniq(allImports, imp) } } if !bst.quietDiscover { mxutil.WriteStr(2, " discover " | pkgPath | " [") for fi, ff := range files { if fi > 0 { mxutil.WriteStr(2, ", ") } mxutil.WriteStr(2, ff) } mxutil.WriteStr(2, "]\n") } version := "" if isRemotePath(pkgPath) && bst.globalModDir != "" { _, reqs, _ := parseModRequires(bst.globalModDir) for _, r := range reqs { if r.path == pkgPath || mxutil.HasPrefix(pkgPath, r.path|"/") { version = r.version break } } } return &pkgInfo{ path: pkgPath, name: pkgName, dir: dir, files: files, cfiles: cFiles, bcfiles: bcFiles, imports: allImports, version: version, } } type resolver struct { visited map[string]bool order []*pkgInfo root string } func (r *resolver) walk(path string) { if r.visited[path] { return } r.visited[path] = true if isBuiltinOnly(path) { return } pkg := discoverPkg(path, r.root) if pkg == nil { return } for _, imp := range pkg.imports { r.walk(imp) } push(r.order, pkg) } func resolveAll(rootPkg string, root string) (ss []*pkgInfo) { r := &resolver{ visited: map[string]bool{}, order: []*pkgInfo{:0:64}, root: root, } r.walk(rootPkg) return r.order } func registerBuiltins() { for k := range cctx.universe.Registry { delete(cctx.universe.Registry, k) } // unsafe - compiler builtin, no source unsafePkg := NewTCPackage("unsafe", "unsafe") unsafePkg.Scope.Insert(NewTypeName(unsafePkg, "Pointer", Typ[UnsafePointer])) unsafePkg.Scope.Insert(NewTCFunc(unsafePkg, "Sizeof", parseSignatureDesc("interface{}->int32"))) unsafePkg.Scope.Insert(NewTCFunc(unsafePkg, "Offsetof", parseSignatureDesc("interface{}->int32"))) unsafePkg.Scope.Insert(NewTCFunc(unsafePkg, "Alignof", parseSignatureDesc("interface{}->int32"))) unsafePkg.Scope.Insert(NewTCFunc(unsafePkg, "Slice", parseSignatureDesc("ptr,int32->[]uint8"))) unsafePkg.Scope.Insert(NewTCFunc(unsafePkg, "SliceData", parseSignatureDesc("[]uint8->ptr"))) cctx.universe.Registry["unsafe"] = unsafePkg // moxie - spawn/codec runtime, types and constants from protocol header mroot := getenv("MOXIEROOT") if mroot == "" { mroot = "." } if !loadMxh(mxutil.JoinPath(mxutil.JoinPath(mxutil.JoinPath(mroot, "src"), "moxie"), "moxie.mxh")) { fatal("cannot load src/moxie/moxie.mxh (MOXIEROOT=" | mroot | ")") } // runtime - assembly + C, pre-compiled in runtime.bc rtPkg := NewTCPackage("runtime", "runtime") rtPkg.Scope.Insert(NewTCFunc(rtPkg, "InitCShared", parseSignatureDesc(""))) rtPkg.Scope.Insert(NewTCFunc(rtPkg, "LastSpawnedParentFd", parseSignatureDesc("->int32"))) rtPkg.Scope.Insert(NewTCFunc(rtPkg, "PipeClosed", parseSignatureDesc("int32->bool"))) rtPkg.Scope.Insert(NewTCFunc(rtPkg, "PipeFDCanSend", parseSignatureDesc("int32->bool"))) rtPkg.Scope.Insert(NewTCFunc(rtPkg, "LockOSThread", parseSignatureDesc(""))) rtPkg.Scope.Insert(NewTCFunc(rtPkg, "UnlockOSThread", parseSignatureDesc(""))) rtPkg.Scope.Insert(NewTCFunc(rtPkg, "KeepAlive", parseSignatureDesc("interface{}"))) rtPkg.Scope.Insert(NewTCFunc(rtPkg, "GOROOT", parseSignatureDesc("->string"))) rtPkg.Scope.Insert(NewTCFunc(rtPkg, "Gosched", parseSignatureDesc(""))) rtPkg.Scope.Insert(NewTCFunc(rtPkg, "Version", parseSignatureDesc("->string"))) rtPkg.Scope.Insert(NewTCVar(rtPkg, "GOOS", Typ[TCString])) rtPkg.Scope.Insert(NewTCVar(rtPkg, "GOARCH", Typ[TCString])) rtPkg.Scope.Insert(NewTCVar(rtPkg, "ChildPipeFd", Typ[Int32])) cctx.universe.Registry["runtime"] = rtPkg } func addPtrMethod(pkg *TCPackage, named *Named, name, sigDesc string) { var sig *Signature if sigDesc == "" { sig = NewSignature(nil, nil, nil, false) } else { sig = parseSignatureDesc(sigDesc) } recvType := NewPointer(named) sig = NewSignature(NewTCVar(pkg, "", recvType), sig.Params, sig.Results, sig.Variadic) fn := NewTCFunc(pkg, name, sig) fn.PtrRecv = true named.AddMethod(fn) } func remapErrorLines(msg string) (s string) { if len(cctx.concatSourceMap) == 0 { return msg } lines := mxutil.SplitLines(msg) var out []string for _, line := range lines { push(out, remapOneLine(line)) } return joinStrings(out, "\n") } func remapOneLine(line string) (s string) { if len(cctx.concatSourceMap) == 0 { return line } colonCount := int32(0) firstColon := int32(-1) secondColon := int32(-1) for i := int32(0); i < int32(len(line)); i++ { if line[i] == ':' { colonCount++ if colonCount == 1 { firstColon = i } else if colonCount == 2 { secondColon = i break } } } if secondColon < 0 { return line } numStr := line[firstColon+1 : secondColon] n := int32(0) valid := len(numStr) > 0 for i := int32(0); i < int32(len(numStr)); i++ { if numStr[i] < '0' || numStr[i] > '9' { valid = false break } n = n*10 + int32(numStr[i]-'0') } if !valid { return line } file, srcLine := concatLineToFile(n) if file == "" { return line } return file | ":" | simpleItoa(srcLine) | line[secondColon:] } func compileSource(src []byte, pkgName, triple, pkgDir string) (s string) { cctx.compilePkgDir = pkgDir ir := CompileToIR(src, pkgName, triple) if len(ir) > 0 && ir[0] == ';' { mxutil.WriteStr(2, remapErrorLines(ir)) fatal("compile error for " | pkgName) } // When streaming (irOutputFile set), IR is written to file directly. // Empty ir string is expected - the file has the content. if len(ir) == 0 && cctx.irOutputFile == "" { fatal("empty IR for " | pkgName) } return ir } // mxhBuf accumulates mxh output. Pre-sized buffer with offset write. // No growth, no abandoned arrays under arena allocation. type mxhBuf struct { data []byte off int32 } func (b *mxhBuf) w(s string) { need := b.off + len(s) if need > len(b.data) { newCap := int32(len(b.data)) * 2 if newCap < need { newCap = need + 4096 } nd := []byte{:newCap} for i := int32(0); i < b.off; i++ { nd[i] = b.data[i] } b.data = nd } for j := int32(0); j < len(s); j++ { b.data[b.off+j] = s[j] } b.off += len(s) } func (b *mxhBuf) nl() { if b.off >= len(b.data) { push(b.data, '\n') b.off++ return } b.data[b.off] = '\n' b.off++ } // typeDescStr returns the type descriptor for simple/leaf types. // For compound types that recurse, use typeDescWrite. func typeDescStr(t Type) (s string) { var b mxhBuf b.data = []byte{:256} typeDescWrite(&b, t) return string(b.data[:b.off]) } func typeDescWrite(b *mxhBuf, t Type) { if t == nil { b.w("interface{}") return } switch v := t.(type) { case *Basic: switch v.Kind { case Bool: b.w("bool") case Int8: b.w("int8") case Int16: b.w("int16") case Int32: b.w("int32") case Int64: b.w("int64") case Uint8: b.w("uint8") case Uint16: b.w("uint16") case Uint32: b.w("uint32") case Uint64: b.w("uint64") case Float32: b.w("float32") case Float64: b.w("float64") case TCString: b.w("string") case UnsafePointer: b.w("ptr") default: b.w("int32") } case *Slice: b.w("[]") typeDescWrite(b, v.Elem) case *Array: b.w("[") b.w(simpleItoa64(v.Len)) b.w("]") typeDescWrite(b, v.Elem) case *Pointer: b.w("*") typeDescWrite(b, v.Base) case *TCStruct: b.w("struct{") for i := int32(0); i < v.NumFields(); i++ { if i > 0 { b.w(",") } f := v.Field(i) if f.Anonymous { b.w("@") } b.w(f.Name) b.w(":") typeDescWrite(b, f.Typ) } b.w("}") case *TCInterface: if v.IsEmpty() { b.w("interface{}") return } hasError := false for mi := int32(0); mi < v.NumMethods(); mi++ { if v.Method(mi).Name == "Error" { hasError = true break } } if hasError { b.w("error") } else { b.w("interface{}") } case *TCChan: switch v.Dir { case TCSendOnly: b.w("chan<-:") case TCRecvOnly: b.w("<-chan:") default: b.w("chan:") } typeDescWrite(b, v.Elem) case *TCMap: b.w("map[") typeDescWrite(b, v.Key) b.w("]") typeDescWrite(b, v.Elem) case *Signature: b.w("func(") sigDescWrite(b, v) b.w(")") case *TypeParam: b.w("$") b.w(v.String()) case *Named: if v.Obj != nil { if cctx.typeDescPkgPath != "" && v.Obj.Pkg != nil && v.Obj.Pkg.Path != cctx.typeDescPkgPath { b.w(v.Obj.Pkg.Path) b.w(".") } b.w(v.Obj.Name) return } typeDescWrite(b, v.Under) default: b.w("interface{}") } } func sigDescWrite(b *mxhBuf, sig *Signature) { if sig.Params != nil { for i := int32(0); i < sig.Params.Len(); i++ { if i > 0 { b.w(",") } pt := sig.Params.At(i).Typ if sig.Variadic && i == sig.Params.Len()-1 { if sl, ok := pt.(*Slice); ok { b.w("...") typeDescWrite(b, sl.Elem) continue } } typeDescWrite(b, pt) } } if sig.Results != nil && sig.Results.Len() > 0 { b.w("->") for i := int32(0); i < sig.Results.Len(); i++ { if i > 0 { b.w(",") } typeDescWrite(b, sig.Results.At(i).Typ) } } } // sigDescStr kept for callers that need a string. func sigDescStr(sig *Signature) (s string) { var b mxhBuf b.data = []byte{:256} sigDescWrite(&b, sig) return string(b.data[:b.off]) } func generateMxh(pkg *TCPackage) (s string) { path := pkg.Path name := pkg.Name cctx.typeDescPkgPath = path var b mxhBuf b.data = []byte{:int32(len(pkg.Scope.Names())) * 80 + 4096} b.w("package ") ; b.w(path) ; b.w(" ") ; b.w(name) ; b.nl() for _, ip := range cctx.universe.DirectImportPaths(pkg) { b.w("import ") ; b.w(ip) ; b.nl() } for _, objName := range pkg.Scope.Names() { if len(objName) == 0 { continue } exported := objName[0] >= 'A' && objName[0] <= 'Z' obj := pkg.Scope.Lookup(objName) if obj == nil { continue } if !exported { if tn, ok := obj.(*TypeName); ok { if named, ok2 := tn.Typ.(*Named); ok2 { _, isStruct := named.Under.(*TCStruct) if named.NumMethods() == 0 && !isStruct { continue } } else { continue } } else { continue } } if fn, ok := obj.(*TCFunc); ok { sig := fn.Signature() if sig != nil { b.w("func ") ; b.w(objName) ; b.w(" ") sigDescWrite(&b, sig) ; b.nl() } } else if vr, ok2 := obj.(*TCVar); ok2 { if vr == nil || vr.Typ == nil { continue } b.w("var ") ; b.w(objName) ; b.w(" ") typeDescWrite(&b, vr.Typ) ; b.nl() } else if cn, ok3 := obj.(*TCConst); ok3 { if cn == nil || cn.Typ == nil { continue } b.w("const ") ; b.w(objName) ; b.w(" ") typeDescWrite(&b, cn.Typ) ; b.w(" ") if cn.Val != nil { b.w(cn.Val.String()) } b.nl() } else if tn2, ok4 := obj.(*TypeName); ok4 { mxhWriteTypeName(&b, objName, tn2) } } return string(b.data[:b.off]) } func mxhWriteTypeName(b *mxhBuf, objName string, v *TypeName) { if v == nil { return } typ := v.Typ if named, ok := typ.(*Named); ok { if ui, ok2 := named.Under.(*TCInterface); ok2 && !ui.IsEmpty() { complete := true for mi := int32(0); mi < ui.NumMethods(); mi++ { if ui.Method(mi).Sig == nil { complete = false break } } if complete { b.w("iface ") ; b.w(objName) mxhWriteTParams(b, named.TParams) b.w(" ") mxhWriteIfaceMethods(b, ui) b.nl() } } else { b.w("type ") ; b.w(objName) mxhWriteTParams(b, named.TParams) b.w(" ") if named.Under != nil { typeDescWrite(b, named.Under) } b.nl() if len(named.TParams) == 0 { for _, m := range named.Methods { msig := m.Signature() if msig != nil { ptrRecv := "0" if m.PtrRecv { ptrRecv = "1" } b.w("method ") ; b.w(objName) ; b.w(" ") b.w(m.Name) ; b.w(" ") sigDescWrite(b, msig) ; b.w(" ") b.w(ptrRecv) ; b.nl() } } } } } else if iface, ok3 := typ.(*TCInterface); ok3 { complete := true for mi := int32(0); mi < iface.NumMethods(); mi++ { if iface.Method(mi).Sig == nil { complete = false break } } if complete { b.w("iface ") ; b.w(objName) ; b.w(" ") mxhWriteIfaceMethods(b, iface) b.nl() } } } func mxhWriteTParams(b *mxhBuf, tparams []*TypeParam) { if len(tparams) == 0 { return } b.w("[") for i, tp := range tparams { if i > 0 { b.w(",") } b.w("$") b.w(tp.String()) } b.w("]") } func mxhWriteIfaceMethods(b *mxhBuf, ui *TCInterface) { for mi := int32(0); mi < ui.NumMethods(); mi++ { m := ui.Method(mi) if mi > 0 { b.w(";") } b.w(m.Name) b.w("=") sigDescWrite(b, m.Sig) } } // parseMxhTParams splits "Curve[$P,$Q]" into ("Curve", [TypeParam P, TypeParam Q]). // For plain names like "Foo" it returns ("Foo", nil). func parseMxhTParams(raw string) (name string, tparams []*TypeParam) { bracketIdx := int32(-1) for i := int32(0); i < int32(len(raw)); i++ { if raw[i] == '[' { bracketIdx = i break } } if bracketIdx < 0 { return raw, nil } name = raw[:bracketIdx] inner := raw[bracketIdx+1 : int32(len(raw))-1] parts := splitCommaBalanced(inner) for _, p := range parts { if len(p) > 1 && p[0] == '$' { tpn := p[1:] push(tparams, NewTypeParam(NewTypeName(nil, tpn, nil), nil)) } } return name, tparams } func loadMxh(path string) (ok bool) { data, ok := mxutil.ReadFile(path) if !ok { return false } return loadMxhData(data) } // mxh index-walking primitives. All return index pairs into the original // buffer d. No intermediate string/slice allocations. // mxhNextLine returns the start and end of the next line at or after pos, // and the position after the newline. end == start when no content remains. func mxhNextLine(d string, pos int32) (ls int32, le int32, next int32) { ls = pos i := pos for i < int32(len(d)) && d[i] != '\n' { i++ } le = i next = i if next < int32(len(d)) { next++ } return ls, le, next } // mxhNextWord returns the start and end of the next whitespace-delimited // word within d[from:limit]. Returns (limit, limit) when no word found. func mxhNextWord(d string, from, limit int32) (ws, we int32) { i := from for i < limit && (d[i] == ' ' || d[i] == '\t') { i++ } ws = i for i < limit && d[i] != ' ' && d[i] != '\t' { i++ } we = i return ws, we } // mxhHasPrefix checks whether d[start:end] begins with prefix. func mxhHasPrefix(d string, start, end int32, prefix string) (ok bool) { plen := int32(len(prefix)) if end-start < plen { return false } for i := int32(0); i < plen; i++ { if d[start+i] != prefix[i] { return false } } return true } // mxhEq checks whether d[start:end] equals s. func mxhEq(d string, start, end int32, s string) (eq bool) { if end-start != int32(len(s)) { return false } for i := int32(0); i < end-start; i++ { if d[start+i] != s[i] { return false } } return true } // mxhParseIfaceMethods parses "name1=sig1;name2=sig2" from d[start:end] // without allocating a []string for the semicolon split. func mxhParseIfaceMethods(d string, start, end int32) (methods []*IfaceMethod) { if start >= end { return nil } // Count methods (semicolon-separated). n := int32(1) for i := start; i < end; i++ { if d[i] == ';' { n++ } } methods = []*IfaceMethod{:0:n} seg := start for i := start; i <= end; i++ { if i == end || d[i] == ';' { eqIdx := int32(-1) for k := seg; k < i; k++ { if d[k] == '=' { eqIdx = k break } } if eqIdx > seg { mname := string(d[seg:eqIdx]) msig := parseSignatureDesc(string(d[eqIdx+1 : i])) push(methods, NewTCIfaceMethod(mname, msig)) } seg = i + 1 } } return methods } // mxhCountWords counts space-delimited words in d[from:limit]. func mxhCountWords(d string, from, limit int32) (n int32) { i := from for i < limit { for i < limit && (d[i] == ' ' || d[i] == '\t') { i++ } if i >= limit { break } n++ for i < limit && d[i] != ' ' && d[i] != '\t' { i++ } } return n } func loadMxhData(data []byte) (ok bool) { // Types created here must survive on the root arena - they're stored // in cctx.universe.Registry which outlives any compile/typecheck arena. mxhSavedArena := runtime.CurrentArena() runtime.SetCurrentArena(runtime.RootArena()) d := string(data) dlen := int32(len(d)) if dlen == 0 { runtime.SetCurrentArena(mxhSavedArena) return false } // Parse header: "package " ls, le, pos := mxhNextLine(d, 0) if !mxhHasPrefix(d, ls, le, "package ") { return false } ws, we := mxhNextWord(d, ls+8, le) if ws == we { return false } pkgPath := string(d[ws:we]) ns, ne := mxhNextWord(d, we, le) if ns == ne { return false } pkgName := string(d[ns:ne]) pkg := NewTCPackage(pkgPath, pkgName) // Collect direct imports. var dimps []string scanPos := pos for scanPos < dlen { ls2, le2, next2 := mxhNextLine(d, scanPos) if mxhHasPrefix(d, ls2, le2, "import ") { push(dimps, string(d[ls2+7:le2])) } scanPos = next2 } cctx.universe.DirectImports[pkgPath] = dimps oldScope := cctx.parseTypePkgScope cctx.parseTypePkgScope = pkg.Scope // Pass 1a: register all type and interface names with placeholder types. scanPos = pos for scanPos < dlen { ls2, le2, next2 := mxhNextLine(d, scanPos) scanPos = next2 if mxhHasPrefix(d, ls2, le2, "type ") { ws2, we2 := mxhNextWord(d, ls2+5, le2) if ws2 < we2 { actualName, tparams := parseMxhTParams(string(d[ws2:we2])) tn := NewTypeName(pkg, actualName, nil) named := NewNamed(tn, nil) named.TParams = tparams pkg.Scope.Insert(tn) } } else if mxhHasPrefix(d, ls2, le2, "iface ") { ws2, we2 := mxhNextWord(d, ls2+6, le2) if ws2 < we2 { actualName, tparams := parseMxhTParams(string(d[ws2:we2])) placeholder := NewTCInterface(nil, nil) placeholder.Complete() tn := NewTypeName(pkg, actualName, nil) named := NewNamed(tn, placeholder) named.TParams = tparams pkg.Scope.Insert(tn) } } } // Pass 1b: resolve underlying types and interface methods. scanPos = pos for scanPos < dlen { ls2, le2, next2 := mxhNextLine(d, scanPos) scanPos = next2 if mxhHasPrefix(d, ls2, le2, "type ") { ws2, we2 := mxhNextWord(d, ls2+5, le2) if ws2 >= we2 { continue } ts, te := mxhNextWord(d, we2, le2) actualName, _ := parseMxhTParams(string(d[ws2:we2])) obj := pkg.Scope.Lookup(actualName) if obj == nil { continue } tn, istn := obj.(*TypeName) if !istn { continue } named, isn := tn.Typ.(*Named) if !isn { continue } if ts >= te { named.Under = NewTCStruct(nil, nil) } else { named.Under = parseTypeDesc(string(d[ts:te])) } } else if mxhHasPrefix(d, ls2, le2, "iface ") { ws2, we2 := mxhNextWord(d, ls2+6, le2) if ws2 >= we2 { continue } ifaceName, _ := parseMxhTParams(string(d[ws2:we2])) // Methods descriptor is the second word (may contain semicolons). ms, me := mxhNextWord(d, we2, le2) methods := mxhParseIfaceMethods(d, ms, me) iface := NewTCInterface(methods, nil) iface.Complete() obj := pkg.Scope.Lookup(ifaceName) if obj == nil { continue } if tn, istn := obj.(*TypeName); istn { if named, isn := tn.Typ.(*Named); isn { named.Under = iface } else { tn.Typ = iface } } } } // Pass 2: funcs, methods, vars, consts. scanPos = pos for scanPos < dlen { ls2, le2, next2 := mxhNextLine(d, scanPos) scanPos = next2 if mxhHasPrefix(d, ls2, le2, "func ") { ws2, we2 := mxhNextWord(d, ls2+5, le2) if ws2 >= we2 { continue } fname := string(d[ws2:we2]) ss, se := mxhNextWord(d, we2, le2) if ss < se { sig := parseSignatureDesc(string(d[ss:se])) pkg.Scope.Insert(NewTCFunc(pkg, fname, sig)) } else { sig := parseSignatureDesc("") pkg.Scope.Insert(NewTCFunc(pkg, fname, sig)) } } else if mxhHasPrefix(d, ls2, le2, "var ") { ws2, we2 := mxhNextWord(d, ls2+4, le2) if ws2 >= we2 { continue } vname := string(d[ws2:we2]) ts, te := mxhNextWord(d, we2, le2) if ts < te { typ := parseTypeDesc(string(d[ts:te])) pkg.Scope.Insert(NewTCVar(pkg, vname, typ)) } } else if mxhHasPrefix(d, ls2, le2, "const ") { ws2, we2 := mxhNextWord(d, ls2+6, le2) if ws2 >= we2 { continue } cname := string(d[ws2:we2]) ts, te := mxhNextWord(d, we2, le2) if ts >= te { continue } typ := parseTypeDesc(string(d[ts:te])) // Remaining words after type are the const value. vs, ve := mxhNextWord(d, te, le2) var val ConstVal if vs < ve { // Reassemble value from remaining words. Const values // may contain spaces (string literals). Walk remaining // words and join with space - but use index range to // avoid per-word | concat. valStart := vs valEnd := le2 // trim trailing whitespace for valEnd > valStart && (d[valEnd-1] == ' ' || d[valEnd-1] == '\t') { valEnd-- } cval := string(d[valStart:valEnd]) isNum := true for ci := int32(0); ci < int32(len(cval)); ci++ { ch := cval[ci] if !((ch >= '0' && ch <= '9') || ch == '-' || ch == '+' || ch == 'x' || ch == 'X' || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F') || ch == ' ') { isNum = false break } } if isNum { n := ssaParseInt64(cval) val = &ConstInt{V: n} } else { if b, isb := typ.(*Basic); isb && (b.Kind == TCString || b.Kind == UntypedString) { val = &ConstStr{S: cval} } else { val = &ConstInt{V: 0} } } } else { val = &ConstInt{V: 0} } pkg.Scope.Insert(NewTCConst(pkg, cname, typ, val)) } else if mxhHasPrefix(d, ls2, le2, "method ") { nwords := mxhCountWords(d, ls2+7, le2) if nwords < 3 { continue } w1s, w1e := mxhNextWord(d, ls2+7, le2) w2s, w2e := mxhNextWord(d, w1e, le2) w3s, w3e := mxhNextWord(d, w2e, le2) typeName := string(d[w1s:w1e]) methodName := string(d[w2s:w2e]) sigDesc := string(d[w3s:w3e]) ptrRecv := false if nwords >= 4 { w4s, w4e := mxhNextWord(d, w3e, le2) if mxhEq(d, w4s, w4e, "1") { ptrRecv = true } } if nwords == 3 && (sigDesc == "0" || sigDesc == "1") { ptrRecv = sigDesc == "1" sigDesc = "" } obj := pkg.Scope.Lookup(typeName) if obj == nil { continue } tn, istn := obj.(*TypeName) if !istn { continue } named, isn := tn.Typ.(*Named) if !isn { continue } sig := parseSignatureDesc(sigDesc) var recvType Type = named if ptrRecv { recvType = NewPointer(named) } sig = NewSignature(NewTCVar(pkg, "", recvType), sig.Params, sig.Results, sig.Variadic) fn := NewTCFunc(pkg, methodName, sig) fn.PtrRecv = ptrRecv named.AddMethod(fn) } } cctx.parseTypePkgScope = oldScope cctx.universe.Registry[pkgPath] = pkg runtime.SetCurrentArena(mxhSavedArena) return true } func cachedMxhPath(cacheBase, pkgPath, hash, version string) (s string) { return mxutil.JoinPath(mxutil.JoinPath(cacheBase, safeName(pkgPath)), versionedCacheName(hash, version, ".mxh")) } func removeAll(path string) { entries := listFiles(path, "") if entries == nil { return } for _, e := range entries { os.Remove(mxutil.JoinPath(path, e)) } os.Remove(path) } func walkStdlibPkgs(srcDir, prefix string, out []string) (result []string) { entries := listFiles(srcDir, "") if entries == nil { return out } mxFiles := listFiles(srcDir, ".mx") if len(mxFiles) > 0 && prefix != "" { push(out, prefix) } for _, e := range entries { if e[0] == '.' || e[0] == '_' { continue } sub := mxutil.JoinPath(srcDir, e) subEntries := listFiles(sub, "") if subEntries != nil { var subPrefix string if prefix == "" { subPrefix = e } else { subPrefix = prefix | "/" | e } out = walkStdlibPkgs(sub, subPrefix, out) } } return out } func cmdCacheWarm() { root := getenv("MOXIEROOT") if root == "" { root = "." } srcDir := mxutil.JoinPath(root, "src") mxutil.WriteStr(2, "warming stdlib cache from " | srcDir | " ...\n") pkgPaths := walkStdlibPkgs(srcDir, "", nil) if len(pkgPaths) == 0 { mxutil.WriteStr(2, " no stdlib packages found\n") return } triple := "x86_64-unknown-linux-musleabihf" clang := "clang-22" tmpdir := "/tmp/mxc-build" mkdirAll(tmpdir, 0755) cacheBase := pkgCacheDir() registerBuiltins() compiled := int32(0) skipped := int32(0) failed := int32(0) for _, pkgPath := range pkgPaths { if isBuiltinOnly(pkgPath) { continue } pkg := discoverPkg(pkgPath, root) if pkg == nil { skipped++ continue } hash := pkgHash(pkg) cached := cachedBcPath(cacheBase, pkg.path, hash, pkg.version) cachedMxh := cachedMxhPath(cacheBase, pkg.path, hash, pkg.version) bcOk := mxutil.FileExists(cached) mxhOk := mxutil.FileExists(cachedMxh) if bcOk && mxhOk { skipped++ continue } compilePath := pkg.path if mxutil.HasPrefix(compilePath, "/") || mxutil.HasPrefix(compilePath, "./") || mxutil.HasPrefix(compilePath, "../") { compilePath = pkg.name } src := concatSources(pkg.dir, pkg.files) ir := compileSource(src, compilePath, triple, pkg.dir) llFile := mxutil.JoinPath(tmpdir, safeName(pkg.path) | ".ll") writeFile(llFile, []byte(ir), 0644) bcFile := mxutil.JoinPath(tmpdir, safeName(pkg.path) | ".bc") rc := run([]string{clang, "--target=" | triple, "-emit-llvm", "-c", llFile, "-o", bcFile}) if rc != 0 { mxutil.WriteStr(2, " FAIL: " | pkg.path | "\n") failed++ continue } cdir := mxutil.JoinPath(cacheBase, safeName(pkg.path)) mkdirAll(cdir, 0755) data, rok := mxutil.ReadFile(bcFile) if rok { writeFile(cached, data, 0644) } if len(cctx.lastMxhData) > 0 { writeFile(cachedMxh, []byte(cctx.lastMxhData), 0644) cctx.lastMxhData = "" } compiled++ mxutil.WriteStr(2, " " | pkg.path | "\n") } mxutil.WriteStr(2, "warm done: " | simpleItoa(compiled) | " compiled, " | simpleItoa(skipped) | " cached, " | simpleItoa(failed) | " failed\n") } func cacheClean() { base := pkgCacheDir() mxutil.WriteStr(2, "cleaning cache: " | base | "\n") entries := listFiles(base, "") if entries == nil { mxutil.WriteStr(2, " (empty)\n") return } for _, e := range entries { removeAll(mxutil.JoinPath(base, e)) } os.Remove(base) mxutil.WriteStr(2, " done\n") } func cmdUpdate(rebuildAll bool) { cacheRoot := mxutil.JoinPath(getMoxiePath(), "cache") currentDir := "mxc-" | mxcVersion() entries := listFiles(cacheRoot, "") if entries == nil { mxutil.WriteStr(2, "cache is empty\n") return } for _, e := range entries { if mxutil.HasPrefix(e, "mxc-") && e != currentDir { mxutil.WriteStr(2, " remove old: " | e | "\n") fullPath := mxutil.JoinPath(cacheRoot, e) pkgEntries := listFiles(fullPath, "") if pkgEntries != nil { for _, pe := range pkgEntries { removeAll(mxutil.JoinPath(fullPath, pe)) } } os.Remove(fullPath) } } if !rebuildAll { mxutil.WriteStr(2, "update done\n") return } mxutil.WriteStr(2, "rebuilding all cached packages...\n") mpath := getMoxiePath() root := getenv("MOXIEROOT") if root == "" { root = "." } cacheBase := pkgCacheDir() pkgDirs := listFiles(cacheBase, "") if pkgDirs == nil { mxutil.WriteStr(2, " no packages to rebuild\n") return } triple := "x86_64-unknown-linux-musleabihf" clang := "clang-22" tmpdir := "/tmp/mxc-build" mkdirAll(tmpdir, 0755) registerBuiltins() for _, pkgDir := range pkgDirs { pkgPath := "" for i := 0; i < len(pkgDir); i++ { if pkgDir[i] == '_' { pkgPath = pkgPath | "/" } else { pkgPath = pkgPath | string([]byte{pkgDir[i]}) } } if len(pkgPath) > 0 && pkgPath[0] == '/' { continue } mxutil.WriteStr(2, " rebuild " | pkgPath | "\n") pkg := discoverPkg(pkgPath, root) if pkg == nil { candidate := mxutil.JoinPath(mpath, pkgPath) mxF := listFiles(candidate, ".mx") if len(mxF) > 0 { pkg = discoverPkg(pkgPath, root) } } if pkg == nil { mxutil.WriteStr(2, " skip (source not found)\n") continue } compilePath := pkg.path if compilePath == "." || compilePath == ".." || mxutil.HasPrefix(compilePath, "/") || mxutil.HasPrefix(compilePath, "./") || mxutil.HasPrefix(compilePath, "../") { compilePath = pkg.name } src := concatSources(pkg.dir, pkg.files) ir := compileSource(src, compilePath, triple, pkg.dir) llFile := mxutil.JoinPath(tmpdir, safeName(pkg.path) | ".ll") writeFile(llFile, []byte(ir), 0644) bcFile := mxutil.JoinPath(tmpdir, safeName(pkg.path) | ".bc") rc := run([]string{clang, "--target=" | triple, "-emit-llvm", "-c", llFile, "-o", bcFile}) if rc != 0 { mxutil.WriteStr(2, " FAIL: clang for " | pkg.path | "\n") continue } hash := pkgHash(pkg) cached := cachedBcPath(cacheBase, pkg.path, hash, pkg.version) cdir := mxutil.JoinPath(cacheBase, safeName(pkg.path)) mkdirAll(cdir, 0755) data, rok := mxutil.ReadFile(bcFile) if rok { writeFile(cached, data, 0644) } if len(cctx.lastMxhData) > 0 { cachedMxh := cachedMxhPath(cacheBase, pkg.path, hash, pkg.version) writeFile(cachedMxh, []byte(cctx.lastMxhData), 0644) cctx.lastMxhData = "" } } mxutil.WriteStr(2, "rebuild done\n") } func scanTestFuncs(data []byte) (ss []string) { var funcs []string lines := mxutil.SplitLines(string(data)) for _, line := range lines { line = mxutil.TrimSpace(line) if !mxutil.HasPrefix(line, "func Test") { continue } paren := int32(-1) for i := int32(5); i < int32(len(line)); i++ { if line[i] == '(' { paren = i break } } if paren < 0 { continue } name := line[5:paren] // "func " is 5 chars, so name starts at "Test..." if len(name) == 0 { continue } if name[0] < 'A' || name[0] > 'Z' { continue } rest := line[paren:] if mxutil.HasPrefix(rest, "(t *testing.T)") { push(funcs, name) } } return funcs } func generateTestMain(pkgName string, testFuncs []string) (buf []byte) { var out []byte out = out | ("package " | pkgName | "\n\nimport \"testing\"\n\nfunc main() {\n\ttests := []testing.InternalTest{\n") for _, name := range testFuncs { out = out | ("\t\t{Name: \"" | name | "\", F: " | name | "},\n") } out = out | "\t}\n\ttesting.Main(tests)\n}\n" return out } func cmdTest(args []string) { pkgPath := "" forceRebuild := false verbose := false i := int32(2) for i < int32(len(args)) { if args[i] == "-a" { forceRebuild = true i++ } else if args[i] == "-v" { verbose = true i++ } else { pkgPath = args[i] i++ } } if pkgPath == "" { pkgPath = "." } _ = verbose root := getenv("MOXIEROOT") if root == "" { root = "." } triple := "x86_64-unknown-linux-musleabihf" clang := "clang-22" if mxutil.HasPrefix(pkgPath, "./") || mxutil.HasPrefix(pkgPath, "../") || pkgPath == "." || mxutil.HasPrefix(pkgPath, "/") { absDir := pkgPath if absDir == "." { absDir = "" } bst.globalModPrefix, bst.globalModDir = findModuleRoot(absDir) } registerBuiltins() var testPkgDir string if pkgPath == "." || pkgPath == ".." || mxutil.HasPrefix(pkgPath, "/") || mxutil.HasPrefix(pkgPath, "./") || mxutil.HasPrefix(pkgPath, "../") { testPkgDir = pkgPath } else { testPkgDir = mxutil.JoinPath(root, mxutil.JoinPath("src", pkgPath)) } testFiles := listFiles(testPkgDir, "_test.mx") if len(testFiles) == 0 { mxutil.WriteStr(2, "no test files found in " | testPkgDir | "\n") return } var allTestFuncs []string for _, tf := range testFiles { data, ok := mxutil.ReadFile(mxutil.JoinPath(testPkgDir, tf)) if !ok { continue } funcs := scanTestFuncs(data) for _, fn := range funcs { push(allTestFuncs, fn) } } if len(allTestFuncs) == 0 { mxutil.WriteStr(2, "no Test* functions found\n") return } mxutil.WriteStr(2, "=== test: " | pkgPath | " ===\n") for _, fn := range allTestFuncs { mxutil.WriteStr(2, " found " | fn | "\n") } pkgName := "" for _, tf := range testFiles { data, ok := mxutil.ReadFile(mxutil.JoinPath(testPkgDir, tf)) if ok && pkgName == "" { pkgName = extractPkgName(data) } } if pkgName == "" { pkgName = "main" } tmpdir := "/tmp/mxc-testbuild" mkdirAll(tmpdir, 0755) testMainSrc := generateTestMain(pkgName, allTestFuncs) testMainFile := mxutil.JoinPath(tmpdir, "_testmain.mx") writeFile(testMainFile, testMainSrc, 0644) pkg := discoverPkg(pkgPath, root) if pkg == nil { fatal("cannot find package " | pkgPath) } for _, tf := range testFiles { push(pkg.files, tf) } push(pkg.files, testMainFile) pkg.name = "main" if !hasImportStr(pkg.imports, "testing") { push(pkg.imports, "testing") } pkgs := resolveTestDeps(pkg, root) cacheBase := pkgCacheDir() for _, dp := range pkgs { compilePath := dp.path if compilePath == "." || compilePath == ".." || mxutil.HasPrefix(compilePath, "/") || mxutil.HasPrefix(compilePath, "./") || mxutil.HasPrefix(compilePath, "../") { compilePath = dp.name } if dp == pkg { compilePath = "main" } hash := pkgHash(dp) cachedMxh := cachedMxhPath(cacheBase, dp.path, hash, dp.version) if mxhOk := mxutil.FileExists(cachedMxh); mxhOk && !forceRebuild && dp != pkg { loadMxh(cachedMxh) src := concatSources(dp.dir, dp.files) ScanGenericDecls(src, compilePath) } else { var src []byte if dp == pkg { src = concatTestSources(dp.dir, dp.files, testMainFile) } else { src = concatSources(dp.dir, dp.files) } TypeCheckOnly(src, compilePath, "") } } var allBcFiles []string hasRuntimePkg := false for _, dp := range pkgs { if dp.path == "runtime" { hasRuntimePkg = true } compilePath := dp.path if compilePath == "." || compilePath == ".." || mxutil.HasPrefix(compilePath, "/") || mxutil.HasPrefix(compilePath, "./") || mxutil.HasPrefix(compilePath, "../") { compilePath = dp.name } if dp == pkg { compilePath = "main" } hash := pkgHash(dp) cached := cachedBcPath(cacheBase, dp.path, hash, dp.version) cachedMxh := cachedMxhPath(cacheBase, dp.path, hash, dp.version) if dp != pkg { bcOk := mxutil.FileExists(cached) mxhOk := mxutil.FileExists(cachedMxh) if bcOk && mxhOk && !forceRebuild { mxutil.WriteStr(2, " cached " | dp.path | "\n") loadMxh(cachedMxh) push(allBcFiles, cached) for _, cf := range dp.cfiles { cPath := mxutil.JoinPath(dp.dir, cf) cbcFile := mxutil.JoinPath(tmpdir, safeName(dp.path) | "_" | cf | ".bc") crc := run([]string{clang, "--target=" | triple, "-emit-llvm", "-c", cPath, "-o", cbcFile}) if crc == 0 { push(allBcFiles, cbcFile) } } for _, bf := range dp.bcfiles { push(allBcFiles, mxutil.JoinPath(dp.dir, bf)) } continue } if isStdlib(dp.path) && bcOk && !forceRebuild { mxutil.WriteStr(2, " cached " | dp.path | "\n") csrc := concatSources(dp.dir, dp.files) cdir := mxutil.JoinPath(cacheBase, safeName(dp.path)) mkdirAll(cdir, 0755) TypeCheckOnly(csrc, compilePath, cachedMxh) push(allBcFiles, cached) continue } } mxutil.WriteStr(2, " compile " | dp.path | "\n") var src []byte if dp == pkg { src = concatTestSources(dp.dir, dp.files, testMainFile) } else { src = concatSources(dp.dir, dp.files) } ir := compileSource(src, compilePath, triple, dp.dir) llFile := mxutil.JoinPath(tmpdir, safeName(dp.path) | ".ll") writeFile(llFile, []byte(ir), 0644) bcFile := mxutil.JoinPath(tmpdir, safeName(dp.path) | ".bc") drc := run([]string{clang, "--target=" | triple, "-emit-llvm", "-c", llFile, "-o", bcFile}) if drc != 0 { fatal("clang failed for " | dp.path) } if dp != pkg { cacheDir := mxutil.JoinPath(cacheBase, safeName(dp.path)) mkdirAll(cacheDir, 0755) data, rok := mxutil.ReadFile(bcFile) if rok { writeFile(cached, data, 0644) } if len(cctx.lastMxhData) > 0 { writeFile(cachedMxh, []byte(cctx.lastMxhData), 0644) cctx.lastMxhData = "" } } push(allBcFiles, bcFile) for _, cf := range dp.cfiles { cPath := mxutil.JoinPath(dp.dir, cf) cbcFile := mxutil.JoinPath(tmpdir, safeName(dp.path) | "_" | cf | ".bc") drc = run([]string{clang, "--target=" | triple, "-emit-llvm", "-c", cPath, "-o", cbcFile}) if drc != 0 { fatal("clang failed for " | cf) } push(allBcFiles, cbcFile) } for _, bf := range dp.bcfiles { push(allBcFiles, mxutil.JoinPath(dp.dir, bf)) } } loadSysroot(mxutil.JoinPath(root, "_sysroot.env")) sysroot := mxutil.JoinPath(root, "sysroot") runtimeBc := getenv("RUNTIME_BC") if runtimeBc == "" { runtimeBc = mxutil.JoinPath(sysroot, "runtime.bc") } muslDir := getenv("MUSL_DIR") if muslDir == "" { muslDir = mxutil.JoinPath(sysroot, "musl") } comprtLib := getenv("COMPRT_LIB") if comprtLib == "" { comprtLib = mxutil.JoinPath(mxutil.JoinPath(sysroot, "compiler-rt"), "lib.a") } muslLib := getenv("MUSL_LIB") if muslLib == "" { muslLib = mxutil.JoinPath(mxutil.JoinPath(sysroot, "musl"), "lib.a") } mxutil.WriteStr(2, " generate initAll\n") var initCalls string for _, dp := range pkgs { compilePath := dp.path if compilePath == "." || compilePath == ".." || mxutil.HasPrefix(compilePath, "/") || mxutil.HasPrefix(compilePath, "./") || mxutil.HasPrefix(compilePath, "../") { compilePath = dp.name } if dp == pkg { continue } initSym := irGlobalSymbol(compilePath, "main") initCalls = initCalls | " call void " | initSym | "(ptr null)\n" } initAllIR := "\ndefine dso_local void @runtime.initAll(ptr %_ctx) {\nentry:\n" | initCalls | " ret void\n}\n" | "declare ptr @llvm.stacksave.p0()\n" | "define ptr @runtime.stacksave(ptr %_ctx) {\n" | " %1 = call ptr @llvm.stacksave.p0()\n" | " ret ptr %1\n" | "}\n" | "define void @runtime.relocCallFixup(i64 %fn, ptr %obj, ptr %rs, ptr %ctx) {\n" | " %fp = inttoptr i64 %fn to ptr\n" | " call void %fp(ptr %obj, ptr %rs, ptr null)\n" | " ret void\n" | "}\n" declared := map[string]bool{} for _, dp := range pkgs { compilePath := dp.path if compilePath == "." || compilePath == ".." || mxutil.HasPrefix(compilePath, "/") || mxutil.HasPrefix(compilePath, "./") || mxutil.HasPrefix(compilePath, "../") { compilePath = dp.name } if dp == pkg { continue } declSym := irGlobalSymbol(compilePath, "main") if declared[declSym] { continue } declared[declSym] = true initAllIR = initAllIR | "declare void " | declSym | "(ptr)\n" } initAllFile := mxutil.JoinPath(tmpdir, "initall.ll") writeFile(initAllFile, []byte(initAllIR), 0644) initAllBc := mxutil.JoinPath(tmpdir, "initall.bc") rc := run([]string{clang, "--target=" | triple, "-emit-llvm", "-c", initAllFile, "-o", initAllBc}) if rc != 0 { fatal("clang failed for initall") } mxutil.WriteStr(2, " compile objects\n") var objList []string if !hasRuntimePkg { runtimeGoBc := mxutil.JoinPath(sysroot, "runtime_go.bc") if _, gok := mxutil.ReadFile(runtimeGoBc); gok { push(objList, clangCompileToObj(clang, triple,runtimeGoBc)) } else { push(objList, clangCompileToObj(clang, triple,runtimeBc)) } } if !hasRuntimePkg { rtShims := mxutil.JoinPath(sysroot, "rt_shims.bc") if _, rsk := mxutil.ReadFile(rtShims); rsk { push(objList, clangCompileToObj(clang, triple,rtShims)) } } if bst.buildJobs > 1 { var objCmds [][]string for _, bf := range allBcFiles { push(objCmds, []string{clang, "--target=" | triple, "-c", "-O0", bf, "-o", bf | ".o"}) } if !runPool(objCmds, bst.buildJobs) { fatal("clang -c failed") } for _, bf := range allBcFiles { push(objList, bf|".o") } } else { for _, bf := range allBcFiles { push(objList, clangCompileToObj(clang, triple,bf)) } } push(objList, clangCompileToObj(clang, triple,initAllBc)) if !hasRuntimePkg { cstubDir := mxutil.JoinPath(root, "src/runtime") cstubFiles := listFiles(cstubDir, ".c") for _, cf := range cstubFiles { cPath := mxutil.JoinPath(cstubDir, cf) coFile := mxutil.JoinPath(tmpdir, "cstub_" | cf | ".o") rc = run([]string{clang, "--target=" | triple, "-c", "-O0", cPath, "-o", coFile}) if rc == 0 { push(objList, coFile) } } } objDir := mxutil.JoinPath(sysroot, "obj") objEntries := listFiles(objDir, ".bc") if objEntries != nil { for _, e := range objEntries { push(objList, clangCompileToObj(clang, triple,mxutil.JoinPath(objDir, e))) } } oEntries := listFiles(objDir, ".o") if oEntries != nil { for _, e := range oEntries { push(objList, mxutil.JoinPath(objDir, e)) } } outpath := mxutil.JoinPath(tmpdir, "test.bin") mxutil.WriteStr(2, " link\n") linker := "ld.lld-22" ldArgs := []string{linker, "--gc-sections", "-o", outpath, mxutil.JoinPath(muslDir, "crt1.o")} for _, o := range objList { push(ldArgs, o) } push(ldArgs, comprtLib, muslLib, "-z", "stack-size=67108864") rc = run(ldArgs) if rc != 0 { fatal("linker failed") } mxutil.WriteStr(2, " run tests\n") rc = run([]string{outpath}) os.Exit(int32(rc)) } func hasImportStr(imports []string, pkg string) (ok bool) { for _, imp := range imports { if imp == pkg { return true } } return false } func resolveTestDeps(testPkg *pkgInfo, root string) (ss []*pkgInfo) { r := &resolver{ visited: map[string]bool{}, root: root, } for _, imp := range testPkg.imports { r.walk(imp) } push(r.order, testPkg) return r.order } func concatTestSources(dir string, files []string, testMainFile string) (buf []byte) { imports := map[string]bool{} var bodies [][]byte var bodyFiles []string var bodyStartLines []int32 var bodyImports [][]string pkgName := "main" allFiles := []string{:0:len(files) + 1} for _, f := range files { if f == testMainFile { continue } push(allFiles, f) } for _, f := range allFiles { var data []byte var ok bool if mxutil.HasPrefix(f, "/") { data, ok = mxutil.ReadFile(f) } else { data, ok = mxutil.ReadFile(mxutil.JoinPath(dir, f)) } if !ok { continue } lines := mxutil.SplitLines(string(data)) var body []string var fimps []string i := int32(0) firstBodyLine := int32(0) for i < int32(len(lines)) { line := mxutil.TrimSpace(lines[i]) if mxutil.HasPrefix(line, "package ") { i++ continue } if line == "import (" { i++ for i < int32(len(lines)) { imp := mxutil.TrimSpace(lines[i]) i++ if imp == ")" { break } if imp != "" { imports[imp] = true push(fimps, imp) } } continue } if mxutil.HasPrefix(line, "import ") && !mxutil.HasPrefix(line, "import (") { imp := line[7:] imports[imp] = true push(fimps, imp) i++ continue } if len(body) == 0 { firstBodyLine = i + 1 } push(body, lines[i]) i++ } push(bodies, []byte(joinStrings(body, "\n"))) push(bodyFiles, f) push(bodyStartLines, firstBodyLine) push(bodyImports, fimps) } tmData, tmOk := mxutil.ReadFile(testMainFile) if tmOk { lines := mxutil.SplitLines(string(tmData)) var body []string var fimps []string i := int32(0) firstBodyLine := int32(0) for i < int32(len(lines)) { line := mxutil.TrimSpace(lines[i]) if mxutil.HasPrefix(line, "package ") { i++ continue } if line == "import (" { i++ for i < int32(len(lines)) { imp := mxutil.TrimSpace(lines[i]) i++ if imp == ")" { break } if imp != "" { imports[imp] = true push(fimps, imp) } } continue } if mxutil.HasPrefix(line, "import ") && !mxutil.HasPrefix(line, "import (") { imp := line[7:] imports[imp] = true push(fimps, imp) i++ continue } if len(body) == 0 { firstBodyLine = i + 1 } push(body, lines[i]) i++ } push(bodies, []byte(joinStrings(body, "\n"))) push(bodyFiles, "_testmain.mx") push(bodyStartLines, firstBodyLine) push(bodyImports, fimps) } var out []byte out = out | ("package " | pkgName | "\n") if len(imports) > 0 { var impKeys []string for imp := range imports { push(impKeys, imp) } for i := int32(1); i < int32(len(impKeys)); i++ { for j := i; j > 0 && impKeys[j] < impKeys[j-1]; j-- { impKeys[j], impKeys[j-1] = impKeys[j-1], impKeys[j] } } out = out | "import (\n" for _, imp := range impKeys { push(out, '\t') out = out | imp push(out, '\n') } out = out | ")\n" } cctx.concatSourceMap = nil mxutil.ResetConcatImportSegs() outLine := int32(1) for j := int32(0); j < int32(len(out)); j++ { if out[j] == '\n' { outLine++ } } for i, b := range bodies { push(cctx.concatSourceMap, srcSpan{ file: bodyFiles[i], outStart: outLine, srcLine: bodyStartLines[i], }) mxutil.AppendConcatImportSeg(mxutil.ImportSeg{ OutStart: outLine, Specs: bodyImports[i], }) for j := int32(0); j < int32(len(b)); j++ { if b[j] == '\n' { outLine++ } } outLine++ out = out | b push(out, '\n') bodies[i] = nil } bodies = nil imports = nil return out } func main() { // Root arena is the domain root's arena, set up by the runtime. // 2GB virtual - pages committed on touch by the MMU. initCompileCtx() cctx.universe = InitUniverse() SetUniverseGlobals(cctx.universe) initBuildState() bst.buildJobs = int32(4) bst.wasmRtPkgs = map[string]bool{ "runtime": true, "runtime/interrupt": true, "internal/task": true, "internal/itoa": true, "math": true, "math/bits": true, "unicode/utf8": true, "strconv": true, "errors": true, "io": true, "io/fs": true, "os": true, "syscall": true, "time": true, "fmt": true, "path": true, "internal/bytealg": true, "internal/byteorder": true, "internal/goarch": true, "internal/binary": true, "internal/gclayout": true, "internal/oserror": true, "internal/stringslite": true, "internal/testlog": true, } bst.builtinOnly = []string{"unsafe"} args := os.Args if len(args) < 2 { mxutil.WriteStr(2, "usage: mxc ...\n") os.Exit(1) } cmd := args[1] if cmd == "version" { mxutil.WriteStr(1, "mxc " | mxcVersion() | " (" | compilerHash() | ")\n") return } if cmd == "cache" { if len(args) >= 3 && args[2] == "clean" { cacheClean() return } if len(args) >= 3 && args[2] == "warm" { cmdCacheWarm() return } mxutil.WriteStr(2, "usage: mxc cache \n") os.Exit(1) } if cmd == "fetch" { fetchURL := "" if len(args) >= 3 { fetchURL = args[2] } cmdFetch(fetchURL) return } if cmd == "update" { rebuildAll := false for ai := int32(2); ai < int32(len(args)); ai++ { if args[ai] == "--all" { rebuildAll = true } } cmdUpdate(rebuildAll) return } if cmd == "test" { cmdTest(args) return } if cmd == "build-runtime" { cmdBuildRuntime() return } if cmd != "build" { mxutil.WriteStr(2, "usage: mxc ...\n") os.Exit(1) } outpath := "" pkgPath := "" printIR := false forceRebuild := false targetFlag := "" i := int32(2) for i < int32(len(args)) { if args[i] == "-o" && i+1 < int32(len(args)) { outpath = args[i+1] i += 2 } else if args[i] == "-target" && i+1 < int32(len(args)) { targetFlag = args[i+1] i += 2 } else if args[i] == "-print-ir" { printIR = true i++ } else if args[i] == "-a" { forceRebuild = true i++ } else if args[i] == "-j" && i+1 < int32(len(args)) { bst.buildJobs = parseInt32(args[i+1]) if bst.buildJobs < 1 { bst.buildJobs = 1 } i += 2 } else if args[i] == "-gc=dealloc" || args[i] == "-buildmode=c-shared" || args[i] == "-x" || args[i] == "-work" { i++ } else { pkgPath = args[i] i++ } } if printIR { // -print-ir needs the in-process compile path. bst.buildJobs = 1 } if pkgPath == "" { fatal("no package specified") } if outpath == "" { outpath = pathBase(pkgPath) } root := getenv("MOXIEROOT") if root == "" { root = "." } mxutil.SetTarget("linux", "amd64") triple := "x86_64-unknown-linux-musleabihf" if targetFlag == "js/wasm" || targetFlag == "wasm" { mxutil.SetTarget("js", "wasm") triple = "wasm32-unknown-wasi" } else if targetFlag == "wasi/wasm" { mxutil.SetTarget("wasi", "wasm") triple = "wasm32-unknown-wasi" } isWasm := mxutil.TargetArch == "wasm" clang := "clang-22" if mxutil.HasPrefix(pkgPath, "./") || mxutil.HasPrefix(pkgPath, "../") || pkgPath == "." || mxutil.HasPrefix(pkgPath, "/") { absDir := pkgPath if absDir == "." { absDir = "" } bst.globalModPrefix, bst.globalModDir = findModuleRoot(absDir) } registerBuiltins() pkgs := resolveAll(pkgPath, root) // wasm: runtime must be compiled from source (no prebuilt runtime.bc) if isWasm { hasRT := false for _, p := range pkgs { if p.path == "runtime" { hasRT = true break } } if !hasRT { rtPkgs := resolveAll("runtime", root) // prepend runtime deps before user packages var merged []*pkgInfo seen := map[string]bool{} for _, p := range rtPkgs { if !seen[p.path] { seen[p.path] = true push(merged, p) } } for _, p := range pkgs { if !seen[p.path] { seen[p.path] = true push(merged, p) } } pkgs = merged } } if len(pkgs) == 0 { fatal("no packages to compile") } tmpdir := "/tmp/mxc-build" mkdirAll(tmpdir, 0755) cacheBase := pkgCacheDir() // TLS probe: if this binary's globals aren't thread_local, the parallel // path would corrupt shared state. Fall back to serial. if bst.buildJobs > 1 && cDetectTLS() == 0 { mxutil.WriteStr(2, " note: globals not TLS-isolated, forcing -j 1 (rebuild with self-compiled mxc for parallel)\n") bst.buildJobs = 1 } var allBcFiles []string var hasRuntimePkg bool if bst.buildJobs > 1 { a, h := buildPkgsParallel(pkgs, triple, clang, tmpdir, cacheBase, forceRebuild, bst.buildJobs) allBcFiles = a hasRuntimePkg = h } else { a, h := buildPkgsSerial(pkgs, triple, clang, tmpdir, cacheBase, forceRebuild, printIR) allBcFiles = a hasRuntimePkg = h } loadSysroot(mxutil.JoinPath(root, "_sysroot.env")) sysroot := mxutil.JoinPath(root, "sysroot") runtimeBc := "" muslDir := "" comprtLib := "" muslLib := "" if !isWasm { runtimeBc = getenv("RUNTIME_BC") if runtimeBc == "" { runtimeBc = mxutil.JoinPath(sysroot, "runtime.bc") } muslDir = getenv("MUSL_DIR") if muslDir == "" { muslDir = mxutil.JoinPath(sysroot, "musl") } comprtLib = getenv("COMPRT_LIB") if comprtLib == "" { comprtLib = mxutil.JoinPath(mxutil.JoinPath(sysroot, "compiler-rt"), "lib.a") } muslLib = getenv("MUSL_LIB") if muslLib == "" { muslLib = mxutil.JoinPath(mxutil.JoinPath(sysroot, "musl"), "lib.a") } } mxutil.WriteStr(2, " generate initAll\n") var initCalls string for _, pkg := range pkgs { compilePath := pkg.path if compilePath == "." || compilePath == ".." || mxutil.HasPrefix(compilePath, "/") || mxutil.HasPrefix(compilePath, "./") || mxutil.HasPrefix(compilePath, "../") { compilePath = pkg.name } if compilePath == "main" { continue } initSym := irGlobalSymbol(compilePath, "main") initCalls = initCalls | " call void " | initSym | "(ptr null)\n" } initAllIR := "\ndefine dso_local void @runtime.initAll(ptr %_ctx) {\nentry:\n" | initCalls | " ret void\n}\n" | "declare ptr @llvm.stacksave.p0()\n" | "define ptr @runtime.stacksave(ptr %_ctx) {\n" | " %1 = call ptr @llvm.stacksave.p0()\n" | " ret ptr %1\n" | "}\n" | "define void @runtime.relocCallFixup(i64 %fn, ptr %obj, ptr %rs, ptr %ctx) {\n" | " %fp = inttoptr i64 %fn to ptr\n" | " call void %fp(ptr %obj, ptr %rs, ptr null)\n" | " ret void\n" | "}\n" declared := map[string]bool{} for _, pkg := range pkgs { compilePath := pkg.path if compilePath == "." || compilePath == ".." || mxutil.HasPrefix(compilePath, "/") || mxutil.HasPrefix(compilePath, "./") || mxutil.HasPrefix(compilePath, "../") { compilePath = pkg.name } if compilePath == "main" { continue } declSym := irGlobalSymbol(compilePath, "main") if declared[declSym] { continue } declared[declSym] = true initAllIR = initAllIR | "declare void " | declSym | "(ptr)\n" } initAllFile := mxutil.JoinPath(tmpdir, "initall.ll") writeFile(initAllFile, []byte(initAllIR), 0644) initAllBc := mxutil.JoinPath(tmpdir, "initall.bc") rc := run([]string{clang, "--target=" | triple, "-emit-llvm", "-c", initAllFile, "-o", initAllBc}) if rc != 0 { fatal("clang failed for initall") } if isWasm { // wasm: compile each .bc to .o individually, then wasm-ld mxutil.WriteStr(2, " compile objects\n") wasmTriple := "wasm32-unknown-wasi" var objList []string if bst.buildJobs > 1 { var objCmds [][]string for _, bf := range allBcFiles { push(objCmds, []string{clang, "--target=" | wasmTriple, "-c", bf, "-o", bf | ".o"}) } if !runPool(objCmds, bst.buildJobs) { fatal("clang -c failed") } for _, bf := range allBcFiles { push(objList, bf|".o") } } else { for _, bf := range allBcFiles { push(objList, clangCompileToObj(clang, wasmTriple,bf)) } } push(objList, clangCompileToObj(clang, wasmTriple,initAllBc)) // Stubs for symbols not provided by stage4-compiled runtime wasiStubs := mxutil.JoinPath(tmpdir, "wasi_stubs.ll") writeFile(wasiStubs, []byte( "target datalayout = \"e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20\"\n" | "target triple = \"wasm32-unknown-wasi\"\n" | "declare ptr @llvm.stacksave.p0()\n" | "define ptr @runtime.stacksave(ptr %_ctx) {\n" | " %1 = call ptr @llvm.stacksave.p0()\n" | " ret ptr %1\n" | "}\n" | "define void @moxie_longjmp(ptr %0, ptr %1) {\n" | " call void @llvm.trap()\n" | " unreachable\n" | "}\n" | "define i32 @\"time.timezoneOffsetMinutes\"() {\n" | " ret i32 0\n" | "}\n" | "declare ptr @runtime.alloc(i32, ptr, ptr)\n" | "define ptr @malloc(i32 %size, ptr %_ctx) {\n" | " %p = call ptr @runtime.alloc(i32 %size, ptr null, ptr null)\n" | " ret ptr %p\n" | "}\n" | "define i32 @strlen(ptr %s, ptr %_ctx) {\n" | "entry:\n" | " br label %loop\n" | "loop:\n" | " %i = phi i32 [0, %entry], [%i1, %next]\n" | " %cp = getelementptr i8, ptr %s, i32 %i\n" | " %c = load i8, ptr %cp\n" | " %z = icmp eq i8 %c, 0\n" | " br i1 %z, label %done, label %next\n" | "next:\n" | " %i1 = add i32 %i, 1\n" | " br label %loop\n" | "done:\n" | " ret i32 %i\n" | "}\n" | "declare void @llvm.trap()\n"), 0644) push(objList, clangCompileToObj(clang, wasmTriple,wasiStubs)) mxutil.WriteStr(2, " link\n") ldArgs := []string{"wasm-ld", "--export-dynamic", "--allow-undefined", "--gc-sections", "-o", outpath} for _, o := range objList { push(ldArgs, o) } push(ldArgs, "-z", "stack-size=67108864") rc = run(ldArgs) if rc != 0 { fatal("linker failed") } } else { // native: compile each .bc to .o, then ld.lld mxutil.WriteStr(2, " compile objects\n") var objList []string if !hasRuntimePkg { runtimeGoBc := mxutil.JoinPath(sysroot, "runtime_go.bc") if _, gok := mxutil.ReadFile(runtimeGoBc); gok { push(objList, clangCompileToObj(clang, triple,runtimeGoBc)) } else { push(objList, clangCompileToObj(clang, triple,runtimeBc)) } } rtShims := mxutil.JoinPath(sysroot, "rt_shims.bc") if _, rsk := mxutil.ReadFile(rtShims); rsk { push(objList, clangCompileToObj(clang, triple,rtShims)) } if bst.buildJobs > 1 { var objCmds [][]string for _, bf := range allBcFiles { push(objCmds, []string{clang, "--target=" | triple, "-c", "-O0", bf, "-o", bf | ".o"}) } if !runPool(objCmds, bst.buildJobs) { fatal("clang -c failed") } for _, bf := range allBcFiles { push(objList, bf|".o") } } else { for _, bf := range allBcFiles { push(objList, clangCompileToObj(clang, triple,bf)) } } push(objList, clangCompileToObj(clang, triple,initAllBc)) if !hasRuntimePkg { cstubDir := mxutil.JoinPath(root, "src/runtime") cstubFiles := listFiles(cstubDir, ".c") for _, cf := range cstubFiles { cPath := mxutil.JoinPath(cstubDir, cf) coFile := mxutil.JoinPath(tmpdir, "cstub_" | cf | ".o") rc = run([]string{clang, "--target=" | triple, "-c", "-O0", cPath, "-o", coFile}) if rc == 0 { push(objList, coFile) } } } objDir := mxutil.JoinPath(sysroot, "obj") objEntries := listFiles(objDir, ".bc") if objEntries != nil { for _, e := range objEntries { push(objList, clangCompileToObj(clang, triple,mxutil.JoinPath(objDir, e))) } } oEntries := listFiles(objDir, ".o") if oEntries != nil { for _, e := range oEntries { push(objList, mxutil.JoinPath(objDir, e)) } } mxutil.WriteStr(2, " link\n") ldArgs := []string{"ld.lld-22", "--gc-sections", "-o", outpath, mxutil.JoinPath(muslDir, "crt1.o")} for _, o := range objList { push(ldArgs, o) } push(ldArgs, comprtLib, muslLib, "-z", "stack-size=67108864") rc = run(ldArgs) if rc != 0 { fatal("linker failed") } } mxutil.WriteStr(2, " -> " | outpath | "\n") } // buildPkgsSerial type-checks and compiles each package in dependency order // in a single pass. Each package's working set dies before the next begins. func buildPkgsSerial(pkgs []*pkgInfo, triple, clang, tmpdir, cacheBase string, forceRebuild, printIR bool) (bcs []string, hasRT bool) { var deferredClang [][]string isWasmTarget := mxutil.TargetArch == "wasm" for _, pkg := range pkgs { if pkg.path == "runtime" { if !isWasmTarget { hasRT = true } } compilePath := pkg.path if compilePath == "." || compilePath == ".." || mxutil.HasPrefix(compilePath, "/") || mxutil.HasPrefix(compilePath, "./") || mxutil.HasPrefix(compilePath, "../") { compilePath = pkg.name } hash := pkgHash(pkg) cached := cachedBcPath(cacheBase, pkg.path, hash, pkg.version) cachedMxh := cachedMxhPath(cacheBase, pkg.path, hash, pkg.version) bcOk := mxutil.FileExists(cached) mxhOk := mxutil.FileExists(cachedMxh) if bcOk && mxhOk && !forceRebuild { mxutil.WriteStr(2, " cached " | pkg.path) if pkg.version != "" { mxutil.WriteStr(2, " @ " | pkg.version) } mxutil.WriteStr(2, "\n") loadMxh(cachedMxh) gsrc := concatSources(pkg.dir, pkg.files) ScanGenericDecls(gsrc, compilePath) push(bcs, cached) for _, cf := range pkg.cfiles { cPath := mxutil.JoinPath(pkg.dir, cf) cbcFile := mxutil.JoinPath(tmpdir, safeName(pkg.path) | "_" | cf | ".bc") crc := run([]string{clang, "--target=" | triple, "-emit-llvm", "-c", cPath, "-o", cbcFile}) if crc == 0 { push(bcs, cbcFile) } } for _, bf := range pkg.bcfiles { push(bcs, mxutil.JoinPath(pkg.dir, bf)) } continue } if isStdlib(pkg.path) && bcOk && !forceRebuild { mxutil.WriteStr(2, " cached " | pkg.path | "\n") csrc := concatSources(pkg.dir, pkg.files) cdir := mxutil.JoinPath(cacheBase, safeName(pkg.path)) mkdirAll(cdir, 0755) TypeCheckOnly(csrc, compilePath, cachedMxh) if cctx.compileArena != nil { runtime.ArenaFree(cctx.compileArena) cctx.compileArena = nil } push(bcs, cached) continue } mxutil.WriteStr(2, " compile " | pkg.path | "\n") src := concatSources(pkg.dir, pkg.files) llFile := mxutil.JoinPath(tmpdir, safeName(pkg.path) | ".ll") // Stream IR directly to .ll file to avoid accumulating entire // package IR in memory (unicode/tables.mx produces ~10MB IR). cctx.irOutputFile = llFile ir := compileSource(src, compilePath, triple, pkg.dir) cctx.irOutputFile = "" if printIR && len(ir) > 0 { mxutil.WriteStr(1, ir) } if len(ir) > 0 { // Non-streaming fallback (error messages start with ';') writeFile(llFile, []byte(ir), 0644) } bcFile := mxutil.JoinPath(tmpdir, safeName(pkg.path) | ".bc") // Defer clang call - collect for parallel execution after all IR emitted. push(deferredClang, []string{clang, "--target=" | triple, "-emit-llvm", "-c", llFile, "-o", bcFile}) // mxh written inside CompileToIR's arena before it dies. if len(cctx.lastMxhData) > 0 { cdir := mxutil.JoinPath(cacheBase, safeName(pkg.path)) mkdirAll(cdir, 0755) writeFile(cachedMxh, []byte(cctx.lastMxhData), 0644) cctx.lastMxhData = "" } push(bcs, bcFile) for _, cf := range pkg.cfiles { cPath := mxutil.JoinPath(pkg.dir, cf) cbcFile := mxutil.JoinPath(tmpdir, safeName(pkg.path) | "_" | cf | ".bc") push(deferredClang, []string{clang, "--target=" | triple, "-emit-llvm", "-c", cPath, "-o", cbcFile}) // rc = run(...) -- deferred if false { fatal("clang failed for " | cf) } push(bcs, cbcFile) } for _, bf := range pkg.bcfiles { push(bcs, mxutil.JoinPath(pkg.dir, bf)) } } // Run all deferred clang calls in parallel. if len(deferredClang) > 0 { nproc := int32(8) if int32(len(deferredClang)) < nproc { nproc = int32(len(deferredClang)) } mxutil.WriteStr(2, " clang: " | simpleItoa(len(deferredClang)) | " files, " | simpleItoa(nproc) | " parallel\n") if !runPool(deferredClang, nproc) { fatal("clang failed") } // Cache the .bc files for _, pkg := range pkgs { compilePath := pkg.path if compilePath == "." || compilePath == ".." || mxutil.HasPrefix(compilePath, "/") || mxutil.HasPrefix(compilePath, "./") || mxutil.HasPrefix(compilePath, "../") { compilePath = pkg.name } hash := pkgHash(pkg) cached := cachedBcPath(cacheBase, compilePath, hash, pkg.version) bcFile := mxutil.JoinPath(tmpdir, safeName(pkg.path) | ".bc") if mxutil.FileExists(bcFile) { cacheDir := mxutil.JoinPath(cacheBase, safeName(pkg.path)) mkdirAll(cacheDir, 0755) data, rok := mxutil.ReadFile(bcFile) if rok { writeFile(cached, data, 0644) } } } } return bcs, hasRT } // --- Ring buffer helpers --- func ringSend(ring unsafe.Pointer, data []byte) (ok bool) { var dp unsafe.Pointer if len(data) > 0 { dp = unsafe.Pointer(unsafe.SliceData(data)) } for { rc := cRingSend(ring, dp, int32(len(data))) if rc == 0 { return true } if rc < 0 { return false } cUsleep(100) // backpressure } } func ringRecv(ring unsafe.Pointer) (data []byte, ok bool) { for { pn := cRingPeekLen(ring) if pn > 0 { buf := []byte{:pn} n := cRingRecv(ring, unsafe.Pointer(unsafe.SliceData(buf)), pn) if n > 0 { return buf[:n], true } if n < 0 { return nil, false } } if pn < 0 { return nil, false } cUsleep(100) } } // ringRecvNonblock polls once. Returns (data, true) on message, (nil, true) // if empty but open, (nil, false) if closed+empty. // Allocates a buffer of exactly the right size for the message. func ringRecvNonblock(ring unsafe.Pointer) (data []byte, ok bool) { pn := cRingPeekLen(ring) if pn == 0 { return nil, true // empty but open } if pn < 0 { return nil, false // closed+empty } buf := []byte{:pn} n := cRingRecv(ring, unsafe.Pointer(unsafe.SliceData(buf)), pn) if n > 0 { return buf[:n], true } if n < 0 { return nil, false } return nil, true } func ringSendInt32(ring unsafe.Pointer, v int32) (ok bool) { var b [4]byte b[0] = byte(v >> 24) b[1] = byte(v >> 16) b[2] = byte(v >> 8) b[3] = byte(v) return ringSend(ring, b[:]) } func ringRecvInt32(ring unsafe.Pointer) (v int32, ok bool) { d, rok := ringRecv(ring) if !rok || len(d) < 4 { return -1, false } v = int32(d[0])<<24 | int32(d[1])<<16 | int32(d[2])<<8 | int32(d[3]) return v, true } // --- Package metadata serialization --- // Text format: newline-separated fields. All strings are raw UTF-8. // Layout: // numPkgs\n // triple\n clang\n tmpdir\n cacheBase\n root\n modPrefix\n modDir\n // for each pkg: path\n name\n dir\n version\n // numFiles\n file1\n ... numCfiles\n cf1\n ... numBcfiles\n bf1\n ... // numImports\n imp1\n ... func serializePkgMeta(pkgs []*pkgInfo, triple, clang, tmpdir, cacheBase, root, modPrefix, modDir string) (data []byte) { var b []byte b = b | simpleItoa(len(pkgs)) push(b, '\n') b = b | triple push(b, '\n') b = b | clang push(b, '\n') b = b | tmpdir push(b, '\n') b = b | cacheBase push(b, '\n') b = b | root push(b, '\n') b = b | modPrefix push(b, '\n') b = b | modDir push(b, '\n') for _, p := range pkgs { b = b | p.path push(b, '\n') b = b | p.name push(b, '\n') b = b | p.dir push(b, '\n') b = b | p.version push(b, '\n') b = b | simpleItoa(len(p.files)) push(b, '\n') for _, f := range p.files { b = b | f push(b, '\n') } b = b | simpleItoa(len(p.cfiles)) push(b, '\n') for _, f := range p.cfiles { b = b | f push(b, '\n') } b = b | simpleItoa(len(p.bcfiles)) push(b, '\n') for _, f := range p.bcfiles { b = b | f push(b, '\n') } b = b | simpleItoa(len(p.imports)) push(b, '\n') for _, im := range p.imports { b = b | im push(b, '\n') } } return b } // deserLine reads the next line from data[pos:], returns it and advances pos. func deserLine(data []byte, pos int32) (line string, next int32) { start := pos for pos < int32(len(data)) && data[pos] != '\n' { pos++ } line = string(data[start:pos]) if pos < int32(len(data)) { pos++ // skip \n } return line, pos } func deserInt(data []byte, pos int32) (v int32, next int32) { s, np := deserLine(data, pos) return parseInt32(s), np } type workerMeta struct { pkgs []*pkgInfo triple string clang string tmpdir string cacheBase string root string modPrefix string modDir string } func deserializePkgMeta(data []byte) (m *workerMeta) { pos := int32(0) numPkgs, pos := deserInt(data, pos) m = &workerMeta{} m.triple, pos = deserLine(data, pos) m.clang, pos = deserLine(data, pos) m.tmpdir, pos = deserLine(data, pos) m.cacheBase, pos = deserLine(data, pos) m.root, pos = deserLine(data, pos) m.modPrefix, pos = deserLine(data, pos) m.modDir, pos = deserLine(data, pos) m.pkgs = []*pkgInfo{:numPkgs} for i := int32(0); i < numPkgs; i++ { p := &pkgInfo{} p.path, pos = deserLine(data, pos) p.name, pos = deserLine(data, pos) p.dir, pos = deserLine(data, pos) p.version, pos = deserLine(data, pos) nf, np := deserInt(data, pos) pos = np if nf > 0 { p.files = []string{:nf} for j := int32(0); j < nf; j++ { p.files[j], pos = deserLine(data, pos) } } nc, npc := deserInt(data, pos) pos = npc if nc > 0 { p.cfiles = []string{:nc} for j := int32(0); j < nc; j++ { p.cfiles[j], pos = deserLine(data, pos) } } nb, npb := deserInt(data, pos) pos = npb if nb > 0 { p.bcfiles = []string{:nb} for j := int32(0); j < nb; j++ { p.bcfiles[j], pos = deserLine(data, pos) } } ni, npi := deserInt(data, pos) pos = npi if ni > 0 { p.imports = []string{:ni} for j := int32(0); j < ni; j++ { p.imports[j], pos = deserLine(data, pos) } } m.pkgs[i] = p } return m } // --- Worker entry (runs in a new thread) --- // Receives metadata via ringA, then receives job indices. Compiles each // package, writes .bc + .mxh to cache, sends result on ringB. // ringA/ringB pointers are passed as a 2-element unsafe.Pointer array. type workerRings struct { ringA unsafe.Pointer // parent -> worker (jobs) ringB unsafe.Pointer // worker -> parent (results) } //export mxc_worker_entry func workerEntryExport(arg unsafe.Pointer) { // Initialize this thread's heap and package state (TLS globals start zeroed). // Must happen before any Moxie allocation. allocateHeap() mmaps a fresh // region, initHeap() sets the bump pointer, initAll() runs package inits. runtime.InitCShared() initBuildState() initCompileCtx() wr := (*workerRings)(arg) ringA := wr.ringA ringB := wr.ringB metaData, metaOk := ringRecv(ringA) if !metaOk { return } meta := deserializePkgMeta(metaData) metaData = nil // Set up module context from metadata bst.globalModPrefix = meta.modPrefix bst.globalModDir = meta.modDir bst.quietDiscover = true // Set target globals from triple (needed by registerBuiltins for uintptr sizing) if len(meta.triple) >= 4 && meta.triple[:4] == "wasm" { mxutil.SetTarget("js", "wasm") } else { mxutil.SetTarget("linux", "amd64") } // Initialize this thread's compiler state (TLS globals are zeroed) registerBuiltins() // Build idxByPath for this thread's copy idxByPath := map[string]int32{} for i := int32(0); i < int32(len(meta.pkgs)); i++ { idxByPath[meta.pkgs[i].path] = i } // Compute hashes for all packages (worker's own bst.pkgKeyMemo) hashes := []string{:len(meta.pkgs)} for i := int32(0); i < int32(len(meta.pkgs)); i++ { hashes[i] = pkgHash(meta.pkgs[i]) } // Work loop: receive job index, compile, send result for { idx, ok := ringRecvInt32(ringA) if !ok { break // ring closed, shutdown } if idx < 0 || idx >= int32(len(meta.pkgs)) { workerSendResult(ringB, idx, "invalid package index") continue } pkg := meta.pkgs[idx] errMsg := workerCompilePkg(pkg, hashes[idx], meta, idxByPath, hashes) workerSendResult(ringB, idx, errMsg) } } // workerSendResult sends idx + error text on ringB. // Empty errMsg = success. Non-empty = error description. // Wire format: [idx:4 BE] [errlen:4 BE] [error text] func workerSendResult(ring unsafe.Pointer, idx int32, errMsg string) { elen := int32(len(errMsg)) msg := []byte{:8 + elen} msg[0] = byte(idx >> 24) msg[1] = byte(idx >> 16) msg[2] = byte(idx >> 8) msg[3] = byte(idx) msg[4] = byte(elen >> 24) msg[5] = byte(elen >> 16) msg[6] = byte(elen >> 8) msg[7] = byte(elen) for i := int32(0); i < elen; i++ { msg[8+i] = errMsg[i] } ringSend(ring, msg) } // workerLoadDeps recursively loads .mxh files for a package's imports, // ensuring transitive deps are loaded before the package that references them. func workerLoadDeps(pkg *pkgInfo, meta *workerMeta, idxByPath map[string]int32, hashes []string) (errMsg string) { for _, imp := range pkg.imports { di, ok := idxByPath[imp] if !ok { continue } dep := meta.pkgs[di] compilePath := dep.path if compilePath == "." || compilePath == ".." || mxutil.HasPrefix(compilePath, "/") || mxutil.HasPrefix(compilePath, "./") || mxutil.HasPrefix(compilePath, "../") { compilePath = dep.name } if cctx.universe.Registry[compilePath] != nil { continue } // Recursively load this dep's deps first if depErr := workerLoadDeps(dep, meta, idxByPath, hashes); depErr != "" { return depErr } depHash := hashes[di] cachedMxh := cachedMxhPath(meta.cacheBase, dep.path, depHash, dep.version) if !loadMxh(cachedMxh) { return "missing dep .mxh for " | dep.path } src := concatSources(dep.dir, dep.files) ScanGenericDecls(src, compilePath) } return "" } // workerCompilePkg compiles one package in the worker thread. All deps must // already have .mxh in cache. Returns empty string on success, error text on failure. func workerCompilePkg(pkg *pkgInfo, hash string, meta *workerMeta, idxByPath map[string]int32, hashes []string) (errMsg string) { // Load all dependency .mxh from cache (recursively, transitive deps first) if loadErr := workerLoadDeps(pkg, meta, idxByPath, hashes); loadErr != "" { return loadErr } compilePath := pkg.path if compilePath == "." || compilePath == ".." || mxutil.HasPrefix(compilePath, "/") || mxutil.HasPrefix(compilePath, "./") || mxutil.HasPrefix(compilePath, "../") { compilePath = pkg.name } mxutil.WriteStr(2, " compile " | pkg.path | "\n") src := concatSources(pkg.dir, pkg.files) cctx.compilePkgDir = pkg.dir ir := CompileToIR(src, compilePath, meta.triple) if len(ir) > 0 && ir[0] == ';' { return remapErrorLines(ir) } if len(ir) == 0 { return "empty IR for " | pkg.path } llFile := mxutil.JoinPath(meta.tmpdir, safeName(pkg.path) | ".ll") writeFile(llFile, []byte(ir), 0644) bcFile := mxutil.JoinPath(meta.tmpdir, safeName(pkg.path) | ".bc") rc := run([]string{meta.clang, "--target=" | meta.triple, "-emit-llvm", "-c", llFile, "-o", bcFile}) if rc != 0 { return "clang failed for " | pkg.path } // Cache .bc cacheDir := mxutil.JoinPath(meta.cacheBase, safeName(pkg.path)) mkdirAll(cacheDir, 0755) cached := cachedBcPath(meta.cacheBase, pkg.path, hash, pkg.version) bcData, rok := mxutil.ReadFile(bcFile) if rok { writeFile(cached, bcData, 0644) } // Cache .mxh (already generated inside CompileToIR's arena) if len(cctx.lastMxhData) > 0 { cachedMxh := cachedMxhPath(meta.cacheBase, pkg.path, hash, pkg.version) writeFile(cachedMxh, []byte(cctx.lastMxhData), 0644) cctx.lastMxhData = "" } // Compile C files for _, cf := range pkg.cfiles { cPath := mxutil.JoinPath(pkg.dir, cf) cbcFile := mxutil.JoinPath(meta.tmpdir, safeName(pkg.path) | "_" | cf | ".bc") rc = run([]string{meta.clang, "--target=" | meta.triple, "-emit-llvm", "-c", cPath, "-o", cbcFile}) if rc != 0 { return "clang failed for " | cf } } return "" } // --- Worker info for the orchestrator --- type workerInfo struct { ringA unsafe.Pointer // parent -> worker ringB unsafe.Pointer // worker -> parent handle uint64 status uint64 busy bool busyIdx int32 } // buildPkgsParallel spawns worker threads with ring buffer IPC. Workers // receive package metadata at startup, then job indices. Each worker // independently loads deps from cache, compiles, writes .bc + .mxh to cache. // No shared mutable state - this IS the Moxie spawn model. func buildPkgsParallel(pkgs []*pkgInfo, triple, clang, tmpdir, cacheBase string, forceRebuild bool, jobs int32) (bcs []string, hasRT bool) { n := int32(len(pkgs)) done := []bool{:n} needBuild := []bool{:n} idxByPath := map[string]int32{} for i := int32(0); i < n; i++ { idxByPath[pkgs[i].path] = i } // Phase 1: check cache for i := int32(0); i < n; i++ { pkg := pkgs[i] hash := pkgHash(pkg) cached := cachedBcPath(cacheBase, pkg.path, hash, pkg.version) cachedMxh := cachedMxhPath(cacheBase, pkg.path, hash, pkg.version) bcOk := mxutil.FileExists(cached) mxhOk := mxutil.FileExists(cachedMxh) if bcOk && mxhOk && !forceRebuild { done[i] = true mxutil.WriteStr(2, " cached " | pkg.path) if pkg.version != "" { mxutil.WriteStr(2, " @ " | pkg.version) } mxutil.WriteStr(2, "\n") } else { needBuild[i] = true } } // Phase 1b: compute transitive dep indices for dispatch readiness. // A package is ready when ALL transitive deps (not just direct) are done. transDeps := [][]int32{:n} for i := int32(0); i < n; i++ { visited := map[int32]bool{} stack := []int32{:0:8} for _, imp := range pkgs[i].imports { if j, ok := idxByPath[imp]; ok { push(stack, j) } } for len(stack) > 0 { top := stack[len(stack)-1] stack = stack[:len(stack)-1] if visited[top] { continue } visited[top] = true for _, imp := range pkgs[top].imports { if j, ok := idxByPath[imp]; ok && !visited[j] { push(stack, j) } } } td := []int32{:0:len(visited)} for j := range visited { push(td, j) } transDeps[i] = td } // Phase 2: serialize metadata and spawn workers root := getenv("MOXIEROOT") if root == "" { root = "." } meta := serializePkgMeta(pkgs, triple, clang, tmpdir, cacheBase, root, bst.globalModPrefix, bst.globalModDir) workers := []*workerInfo{:jobs} for wi := int32(0); wi < jobs; wi++ { w := &workerInfo{} w.ringA = cRingCreate(262144) // 256KB jobs+metadata w.ringB = cRingCreate(262144) // 256KB results+errors if w.ringA == nil || w.ringB == nil { fatal("ring_create failed for worker " | simpleItoa(wi)) } // Pack rings into arg struct (lives until thread reads it) wr := &workerRings{ringA: w.ringA, ringB: w.ringB} rc := cSpawnWorker(unsafe.Pointer(wr), unsafe.Pointer(&w.handle), unsafe.Pointer(&w.status)) if rc != 0 { fatal("spawn_worker failed for worker " | simpleItoa(wi)) } // Send metadata to worker via ringA if !ringSend(w.ringA, meta) { fatal("failed to send metadata to worker " | simpleItoa(wi)) } workers[wi] = w } meta = nil // Phase 3: three-check dispatch loop failed := false for { allDone := true for i := int32(0); i < n; i++ { if needBuild[i] && !done[i] { allDone = false break } } if allDone { break } // On failure: close all ringAs to signal workers to stop, drain busy if failed { for wi := int32(0); wi < int32(len(workers)); wi++ { w := workers[wi] if w != nil { cRingClose(w.ringA) // idempotent } } anyBusy := false for wi := int32(0); wi < int32(len(workers)); wi++ { w := workers[wi] if w != nil && w.busy { anyBusy = true break } } if !anyBusy { break } // Fall through to collection to drain in-flight results } if !failed { // Check 1: death check for wi := int32(0); wi < int32(len(workers)); wi++ { w := workers[wi] if w == nil { continue } alive := cThreadAlive(unsafe.Pointer(&w.status)) if alive < 0 { if w.busy { mxutil.WriteStr(2, " CRASHED worker " | simpleItoa(wi) | " compiling " | pkgs[w.busyIdx].path | "\n") } else { mxutil.WriteStr(2, " CRASHED worker " | simpleItoa(wi) | "\n") } failed = true workers[wi] = nil } } // Check 2: dispatch jobs to idle workers if !failed { for wi := int32(0); wi < int32(len(workers)); wi++ { w := workers[wi] if w == nil || w.busy { continue } // Find a dispatchable package di := int32(-1) for i := int32(0); i < n; i++ { if done[i] || !needBuild[i] { continue } // Check not already in flight inFlight := false for wj := int32(0); wj < int32(len(workers)); wj++ { wk := workers[wj] if wk != nil && wk.busy && wk.busyIdx == i { inFlight = true break } } if inFlight { continue } // Check all transitive deps done ready := true for _, j := range transDeps[i] { if !done[j] { ready = false break } } if ready { di = i break } } if di < 0 { continue } // Dispatch if !ringSendInt32(w.ringA, di) { mxutil.WriteStr(2, " dispatch failed for " | pkgs[di].path | "\n") failed = true break } w.busy = true w.busyIdx = di } } } // Check 3: collect results from busy workers (always runs - drains on failure) progress := false for wi := int32(0); wi < int32(len(workers)); wi++ { w := workers[wi] if w == nil || !w.busy { continue } rd, rok := ringRecvNonblock(w.ringB) if rd == nil { if !rok { mxutil.WriteStr(2, " worker " | simpleItoa(wi) | " ringB closed\n") failed = true } continue } if len(rd) < 8 { continue } ridx := int32(rd[0])<<24 | int32(rd[1])<<16 | int32(rd[2])<<8 | int32(rd[3]) elen := int32(rd[4])<<24 | int32(rd[5])<<16 | int32(rd[6])<<8 | int32(rd[7]) w.busy = false progress = true if elen > 0 && int32(len(rd)) >= 8+elen { errText := string(rd[8 : 8+elen]) mxutil.WriteStr(2, errText) mxutil.WriteStr(2, "mxc: compile error for " | pkgs[ridx].path | "\n") failed = true } else if elen > 0 { mxutil.WriteStr(2, "mxc: compile error for " | pkgs[ridx].path | "\n") failed = true } else { done[ridx] = true } } // Join point if !progress { cUsleep(1000) } } // Phase 4: shutdown - close rings, join threads for wi := int32(0); wi < int32(len(workers)); wi++ { w := workers[wi] if w == nil { continue } cRingClose(w.ringA) cThreadJoin(unsafe.Pointer(&w.handle), unsafe.Pointer(&w.status)) cRingDestroy(w.ringA) cRingDestroy(w.ringB) } workers = nil if failed { fatal("parallel build failed") } for i := int32(0); i < n; i++ { if needBuild[i] && !done[i] { fatal("dependency cycle: cannot schedule " | pkgs[i].path) } } // Phase 5: assemble .bc list for i := int32(0); i < n; i++ { pkg := pkgs[i] if pkg.path == "runtime" { hasRT = true } if needBuild[i] { push(bcs, mxutil.JoinPath(tmpdir, safeName(pkg.path) | ".bc")) for _, cf := range pkg.cfiles { push(bcs, mxutil.JoinPath(tmpdir, safeName(pkg.path) | "_" | cf | ".bc")) } } else { hash := pkgHash(pkg) push(bcs, cachedBcPath(cacheBase, pkg.path, hash, pkg.version)) for _, cf := range pkg.cfiles { cPath := mxutil.JoinPath(pkg.dir, cf) cbcFile := mxutil.JoinPath(tmpdir, safeName(pkg.path) | "_" | cf | ".bc") rc := run([]string{clang, "--target=" | triple, "-emit-llvm", "-c", cPath, "-o", cbcFile}) if rc == 0 { push(bcs, cbcFile) } } } for _, bf := range pkg.bcfiles { push(bcs, mxutil.JoinPath(pkg.dir, bf)) } } return bcs, hasRT } func cmdBuildRuntime() { root := getenv("MOXIEROOT") if root == "" { root = "." } triple := "x86_64-unknown-linux-musleabihf" clang := "clang-22" sysroot := mxutil.JoinPath(root, "sysroot") tmpdir := "/tmp/mxc-rt-build" mkdirAll(tmpdir, 0755) registerBuiltins() rtPkgs := []string{"internal/task", "runtime"} var bcFiles []string for _, pkg := range rtPkgs { dir := mxutil.JoinPath(root, mxutil.JoinPath("src", pkg)) info := discoverPkg(pkg, root) if info == nil || len(info.files) == 0 { mxutil.WriteStr(2, " skip " | pkg | " (no files)\n") continue } mxutil.WriteStr(2, " compile " | pkg | "\n") src := concatSources(dir, info.files) ir := compileSource(src, pkg, triple, dir) llFile := mxutil.JoinPath(tmpdir, safeName(pkg) | ".ll") writeFile(llFile, []byte(ir), 0644) bcFile := mxutil.JoinPath(tmpdir, safeName(pkg) | ".bc") rc := run([]string{clang, "--target=" | triple, "-emit-llvm", "-c", llFile, "-o", bcFile}) if rc != 0 { fatal("clang failed for " | pkg) } push(bcFiles, bcFile) } if len(bcFiles) == 0 { fatal("no runtime packages compiled") } outBc := mxutil.JoinPath(sysroot, "runtime_go.bc") if len(bcFiles) == 1 { data, ok := mxutil.ReadFile(bcFiles[0]) if !ok { fatal("cannot read " | bcFiles[0]) } writeFile(outBc, data, 0644) } else { linkArgs := []string{"llvm-link-22", "-o", outBc} for _, bf := range bcFiles { push(linkArgs, bf) } rc := run(linkArgs) if rc != 0 { fatal("llvm-link failed merging runtime") } } mxutil.WriteStr(2, " -> " | outBc | "\n") }