package main import ( "os" "unsafe" ) //export mxc_run func cRun(argv unsafe.Pointer, argc int32) int32 //export mxc_getenv func cGetenv(name unsafe.Pointer, nameLen int32, buf unsafe.Pointer, bufCap int32) int32 //export mxc_listdir func cListdir(dir unsafe.Pointer, dirLen int32, buf unsafe.Pointer, bufCap int32) int32 //export write func cWrite(fd int32, buf unsafe.Pointer, count uint32) int32 //export mxc_writefile func cWritefile(path unsafe.Pointer, pathLen int32, data unsafe.Pointer, dataLen int32, mode uint32) int32 //export mxc_mkdir func cMkdir(path unsafe.Pointer, pathLen int32, mode uint32) int32 //export mxc_readfile func cReadfile(path unsafe.Pointer, pathLen int32, buf unsafe.Pointer, bufCap int32) int32 //export mxc_filesize func cFilesize(path unsafe.Pointer, pathLen int32) int32 func writeStr(fd int32, s string) { b := []byte(s) if len(b) > 0 { cWrite(fd, unsafe.Pointer(unsafe.SliceData(b)), uint32(len(b))) } } func writeFile(path string, data []byte, mode uint32) { if len(path) == 0 { return } cWritefile(unsafe.Pointer(unsafe.SliceData([]byte(path))), int32(len(path)), unsafe.Pointer(unsafe.SliceData(data)), int32(len(data)), mode) } func mkdirAll(path string, mode uint32) { if len(path) == 0 { return } cMkdir(unsafe.Pointer(unsafe.SliceData([]byte(path))), int32(len(path)), mode) } func readFile(path string) ([]byte, bool) { if len(path) == 0 { return nil, false } pb := []byte(path) sz := cFilesize(unsafe.Pointer(unsafe.SliceData(pb)), int32(len(pb))) if sz < 0 { return nil, false } if sz == 0 { sz = 65536 } buf := []byte{:sz} n := cReadfile(unsafe.Pointer(unsafe.SliceData(pb)), int32(len(pb)), unsafe.Pointer(unsafe.SliceData(buf)), sz) if n < 0 { return nil, false } return buf[:n], true } func fatal(msg string) { writeStr(2, "mxc: " | msg | "\n") os.Exit(1) } func getenv(name string) string { nb := []byte(name) buf := []byte{:4096} n := cGetenv(unsafe.Pointer(unsafe.SliceData(nb)), int32(len(nb)), unsafe.Pointer(unsafe.SliceData(buf)), int32(len(buf))) if n <= 0 { return "" } return string(buf[:n]) } func getArgs() []string { data, ok := readFile("/proc/self/cmdline") if !ok { return nil } var args []string start := int32(0) for i := int32(0); i < int32(len(data)); i++ { if data[i] == 0 { if i > start { args = append(args, string(data[start:i])) } start = i + 1 } } if start < int32(len(data)) { args = append(args, string(data[start:])) } return args } func listFiles(dir, ext string) []string { db := []byte(dir) buf := []byte{:65536} n := cListdir(unsafe.Pointer(unsafe.SliceData(db)), int32(len(db)), unsafe.Pointer(unsafe.SliceData(buf)), int32(len(buf))) if n <= 0 { return nil } var result []string start := int32(0) for i := int32(0); i < n; i++ { if buf[i] == 0 { name := string(buf[start:i]) if hasSuffix(name, ext) { result = append(result, name) } start = i + 1 } } return result } func run(args []string) int32 { argv := []unsafe.Pointer{:len(args) + 1} cstrs := [][]byte{:len(args)} for i, a := range args { b := []byte{:len(a) + 1} copy(b, a) b[len(a)] = 0 cstrs[i] = b argv[i] = unsafe.Pointer(unsafe.SliceData(b)) } return cRun(unsafe.Pointer(unsafe.SliceData(argv)), int32(len(args))) } func splitLines(s string) []string { var result []string start := int32(0) for i := int32(0); i < int32(len(s)); i++ { if s[i] == '\n' { if i > start { result = append(result, s[start:i]) } start = i + 1 } } if start < int32(len(s)) { result = append(result, s[start:]) } return result } func hasPrefix(s, prefix string) bool { return len(s) >= len(prefix) && s[:len(prefix)] == prefix } func hasSuffix(s, suffix string) bool { return len(s) >= len(suffix) && s[len(s)-len(suffix):] == suffix } func trimSpace(s string) string { i := int32(0) for i < int32(len(s)) && (s[i] == ' ' || s[i] == '\t' || s[i] == '\n' || s[i] == '\r') { i++ } j := int32(len(s)) for j > i && (s[j-1] == ' ' || s[j-1] == '\t' || s[j-1] == '\n' || s[j-1] == '\r') { j-- } return s[i:j] } func joinPath(a, b string) string { if len(a) == 0 { return b } if a[len(a)-1] == '/' { return a | b } return a | "/" | b } func pathBase(path string) string { for i := int32(len(path)) - 1; i >= 0; i-- { if path[i] == '/' { return path[i+1:] } } return path } func pathDir(path string) string { for i := int32(len(path)) - 1; i >= 0; i-- { if path[i] == '/' { if i == 0 { return "/" } return path[:i] } } return "." } func splitSlash(s string) []string { var parts []string start := int32(0) for i := int32(0); i < int32(len(s)); i++ { if s[i] == '/' { parts = append(parts, s[start:i]) start = i + 1 } } parts = append(parts, s[start:]) return parts } func cleanPath(path string) string { parts := splitSlash(path) var out []string for _, p := range parts { if p == "." || p == "" { continue } if p == ".." && len(out) > 0 && out[len(out)-1] != ".." { out = out[:len(out)-1] } else { out = append(out, p) } } if len(out) == 0 { return "." } result := out[0] for i := 1; i < len(out); i++ { result = result | "/" | out[i] } if len(path) > 0 && path[0] == '/' { result = "/" | result } return result } func getMoxiePath() string { p := getenv("MOXIEPATH") if p != "" { return p } home := getenv("HOME") if home == "" { return "/tmp/moxie" } return joinPath(home, "moxie") } func mxcVersion() string { return "1.7.0" } func moxieCacheDir() string { return joinPath(joinPath(getMoxiePath(), "cache"), "mxc-" | mxcVersion()) } var globalModPrefix string var globalModDir string var globalReplaces map[string]string func parseReplaceLine(line string, baseDir string) (string, string, bool) { arrow := int32(-1) for k := int32(0); k < int32(len(line))-1; k++ { if line[k] == '=' && line[k+1] == '>' { arrow = k break } } if arrow < 0 { return "", "", false } from := trimSpace(line[:arrow]) to := trimSpace(line[arrow+2:]) for fi := int32(0); fi < int32(len(from)); fi++ { if from[fi] == ' ' { from = from[:fi] break } } if hasPrefix(to, "./") || hasPrefix(to, "../") || hasPrefix(to, "/") { to = cleanPath(joinPath(baseDir, to)) } return from, to, true } func findModuleRoot(startDir string) (string, string) { dir := startDir for { data, ok := readFile(joinPath(dir, "moxie.mod")) if ok { modName := "" globalReplaces = map[string]string{} lines := splitLines(string(data)) inReplace := false for _, line := range lines { line = trimSpace(line) if hasPrefix(line, "module ") { modName = trimSpace(line[7:]) continue } if line == "replace (" { inReplace = true continue } if inReplace { if line == ")" { inReplace = false continue } from, to, ok2 := parseReplaceLine(line, dir) if ok2 { globalReplaces[from] = to } continue } if hasPrefix(line, "replace ") { from, to, ok2 := parseReplaceLine(line[8:], dir) if ok2 { globalReplaces[from] = to } } } if modName != "" { return modName, dir } } parent := pathDir(dir) if parent == dir || parent == "" || parent == "." { break } dir = parent } return "", "" } func mergeModReplaces(dir string) { data, ok := readFile(joinPath(dir, "moxie.mod")) if !ok { return } if globalReplaces == nil { globalReplaces = map[string]string{} } inReplace := false for _, line := range splitLines(string(data)) { line = trimSpace(line) if line == "replace (" { inReplace = true continue } if inReplace { if line == ")" { inReplace = false continue } from, to, ok2 := parseReplaceLine(line, dir) if ok2 { if _, exists := globalReplaces[from]; !exists { globalReplaces[from] = to } } continue } if hasPrefix(line, "replace ") { from, to, ok2 := parseReplaceLine(line[8:], dir) if ok2 { if _, exists := globalReplaces[from]; !exists { globalReplaces[from] = to } } } } } type modRequire struct { path string version string } func parseModRequires(dir string) (string, []modRequire, map[string]string) { data, ok := readFile(joinPath(dir, "moxie.mod")) if !ok { return "", nil, nil } modName := "" var reqs []modRequire repls := map[string]string{} inRequire := false inReplace := false for _, line := range splitLines(string(data)) { line = trimSpace(line) if hasPrefix(line, "module ") { modName = trimSpace(line[7:]) continue } if line == "require (" { inRequire = true continue } if inRequire { if line == ")" { inRequire = false continue } parts := splitBySpace(line) if len(parts) >= 2 { reqs = append(reqs, modRequire{path: parts[0], version: parts[1]}) } else if len(parts) == 1 { reqs = append(reqs, modRequire{path: parts[0], version: ""}) } continue } if hasPrefix(line, "require ") { parts := splitBySpace(line[8:]) if len(parts) >= 2 { reqs = append(reqs, modRequire{path: parts[0], version: parts[1]}) } else if len(parts) == 1 { reqs = append(reqs, modRequire{path: parts[0], version: ""}) } continue } if line == "replace (" { inReplace = true continue } if inReplace { if line == ")" { inReplace = false continue } from, to, ok2 := parseReplaceLine(line, dir) if ok2 { repls[from] = to } continue } if hasPrefix(line, "replace ") { from, to, ok2 := parseReplaceLine(line[8:], dir) if ok2 { repls[from] = to } } } return modName, reqs, repls } func isRemotePath(path string) bool { slash := int32(-1) for i := int32(0); i < int32(len(path)); i++ { if path[i] == '/' { slash = i break } } if slash < 0 { return false } host := path[:slash] for i := int32(0); i < int32(len(host)); i++ { if host[i] == '.' { return true } } return false } func repoLockPath(mpath, repoPath string) string { return joinPath(mpath, repoPath | ".lock") } func acquireRepoLock(mpath, repoPath string) bool { lp := repoLockPath(mpath, repoPath) _, locked := readFile(lp) if locked { return false } mkdirAll(pathDir(lp), 0755) writeFile(lp, []byte("1"), 0644) return true } func releaseRepoLock(mpath, repoPath string) { os.Remove(repoLockPath(mpath, repoPath)) } func autoFetchRepo(pkgPath, version string) string { mpath := getMoxiePath() repoPath := pkgPath for { dest := joinPath(mpath, repoPath) gitHead := joinPath(dest, ".git/HEAD") _, exists := readFile(gitHead) if exists { break } parent := pathDir(repoPath) if parent == repoPath || parent == "." || parent == "" { break } parentDest := joinPath(mpath, parent) parentGit := joinPath(parentDest, ".git/HEAD") _, parentExists := readFile(parentGit) if parentExists { repoPath = parent break } repoPath = parent } if !acquireRepoLock(mpath, repoPath) { writeStr(2, " waiting for lock: " | repoPath | "\n") fatal("repo locked: " | repoPath) } dest := joinPath(mpath, repoPath) gitHead := joinPath(dest, ".git/HEAD") _, exists := readFile(gitHead) if !exists { mkdirAll(pathDir(dest), 0755) url := "https://" | repoPath writeStr(2, " auto-fetch " | repoPath | "\n") if run([]string{"git", "clone", url, dest}) != 0 { releaseRepoLock(mpath, repoPath) return "" } } if version != "" && version != "v0.0.0" { if run([]string{"git", "-C", dest, "checkout", version}) != 0 { run([]string{"git", "-C", dest, "fetch", "--tags"}) run([]string{"git", "-C", dest, "pull"}) run([]string{"git", "-C", dest, "checkout", version}) } } releaseRepoLock(mpath, repoPath) return joinPath(mpath, pkgPath) } func cmdFetch() { mpath := getMoxiePath() mkdirAll(mpath, 0755) modName, reqs, repls := parseModRequires(".") if modName == "" { fatal("no moxie.mod in current directory") } writeStr(2, "module: " | modName | "\n") var queue []modRequire for _, r := range reqs { if _, ok := repls[r.path]; ok { writeStr(2, " skip (replaced): " | r.path | "\n") continue } queue = append(queue, r) } seen := map[string]bool{} for len(queue) > 0 { req := queue[0] queue = queue[1:] if seen[req.path] { continue } seen[req.path] = true if !acquireRepoLock(mpath, req.path) { writeStr(2, " locked: " | req.path | " (another build in progress)\n") fatal("repo locked: " | req.path) } dest := joinPath(mpath, req.path) gitHead := joinPath(dest, ".git/HEAD") _, exists := readFile(gitHead) if !exists { mkdirAll(pathDir(dest), 0755) url := "https://" | req.path writeStr(2, " clone " | req.path | "\n") if run([]string{"git", "clone", url, dest}) != 0 { releaseRepoLock(mpath, req.path) writeStr(2, " FAIL: clone " | req.path | "\n") continue } } if req.version != "" && req.version != "v0.0.0" { if run([]string{"git", "-C", dest, "checkout", req.version}) != 0 { writeStr(2, " tag " | req.version | " not found, pulling\n") run([]string{"git", "-C", dest, "pull"}) if run([]string{"git", "-C", dest, "checkout", req.version}) != 0 { writeStr(2, " WARN: " | req.version | " unavailable for " | req.path | "\n") } } else { writeStr(2, " " | req.path | " @ " | req.version | "\n") } } else if exists { writeStr(2, " pull " | req.path | "\n") run([]string{"git", "-C", dest, "pull"}) } else { writeStr(2, " fetched " | req.path | "\n") } releaseRepoLock(mpath, req.path) _, subReqs, _ := parseModRequires(dest) for _, sr := range subReqs { if !seen[sr.path] { queue = append(queue, sr) } } } writeStr(2, "fetch done\n") } func joinStrings(ss []string, sep string) string { if len(ss) == 0 { return "" } n := int32(0) for _, s := range ss { n += int32(len(s)) } n += int32(len(sep)) * (int32(len(ss)) - 1) buf := []byte{:0:n} for i, s := range ss { if i > 0 { buf = append(buf, sep...) } buf = append(buf, s...) } return string(buf) } func appendUniq(ss []string, s string) []string { for _, x := range ss { if x == s { return ss } } return append(ss, s) } func fnvHash(data []byte) string { h := uint32(2166136261) for i := int32(0); i < int32(len(data)); i++ { h ^= uint32(data[i]) h *= uint32(16777619) } hex := "0123456789abcdef" buf := []byte{:8} for i := int32(7); i >= 0; i-- { buf[i] = hex[h&0xf] h >>= 4 } return string(buf) } func pkgCacheDir() string { return joinPath(moxieCacheDir(), "pkg") } func pkgCacheKey(dir string, files []string) string { var sorted []string for _, f := range files { sorted = append(sorted, f) } for i := int32(1); i < int32(len(sorted)); i++ { for j := i; j > 0 && sorted[j] < sorted[j-1]; j-- { sorted[j], sorted[j-1] = sorted[j-1], sorted[j] } } var all []byte for _, f := range sorted { p := joinPath(dir, f) data, ok := readFile(p) if !ok { continue } all = append(all, data...) all = append(all, 0) } return fnvHash(all) } func versionedCacheName(hash, version, ext string) string { if version != "" { return hash | "_" | version | ext } return hash | ext } func cachedBcPath(cacheBase, pkgPath, hash, version string) string { return joinPath(joinPath(cacheBase, safeName(pkgPath)), versionedCacheName(hash, version, ".bc")) } func isStdlib(path string) bool { return !hasPrefix(path, "/") && !hasPrefix(path, "./") && !hasPrefix(path, "../") } var builtinOnly []string func isBuiltinOnly(path string) bool { for _, b := range builtinOnly { if path == b { return true } } return false } func loadSysroot(path string) { data, ok := readFile(path) if !ok { return } for _, line := range splitLines(string(data)) { line = trimSpace(line) if len(line) == 0 || line[0] == '#' { continue } eq := int32(-1) for i := int32(0); i < int32(len(line)); i++ { if line[i] == '=' { eq = i break } } if eq < 0 { continue } key := line[:eq] val := line[eq+1:] os.Setenv(key, val) } } func splitBySpace(s string) []string { var result []string start := int32(0) inWord := false for i := int32(0); i < int32(len(s)); i++ { if s[i] == ' ' || s[i] == '\t' { if inWord { result = append(result, s[start:i]) inWord = false } } else { if !inWord { start = i inWord = true } } } if inWord { result = append(result, s[start:]) } return result } func safeName(pkg string) string { b := []byte{:len(pkg)} for i := int32(0); i < int32(len(pkg)); i++ { if pkg[i] == '/' { b[i] = '_' } else { b[i] = pkg[i] } } return string(b) } func extractPkgName(data []byte) string { for i := int32(0); i < int32(len(data)); i++ { if i+8 < int32(len(data)) && string(data[i:i+8]) == "package " { j := i + 8 for j < int32(len(data)) && data[j] != '\n' && data[j] != '\r' && data[j] != ' ' { j++ } return string(data[i+8 : j]) } if data[i] == '\n' { continue } if data[i] == '/' && i+1 < int32(len(data)) && data[i+1] == '/' { for i < int32(len(data)) && data[i] != '\n' { i++ } continue } break } return "main" } func extractImports(data []byte) []string { var imports []string lines := splitLines(string(data)) inBlock := false for _, line := range lines { line = trimSpace(line) if line == "import (" { inBlock = true continue } if inBlock && line == ")" { inBlock = false continue } if inBlock { imp := extractQuoted(line) if imp != "" { imports = appendUniq(imports, imp) } } if hasPrefix(line, "import \"") { imp := extractQuoted(line[7:]) if imp != "" { imports = appendUniq(imports, imp) } } } return imports } func extractQuoted(s string) string { start := int32(-1) for i := int32(0); i < int32(len(s)); i++ { if s[i] == '"' { if start < 0 { start = i + 1 } else { return s[start:i] } } } return "" } func concatSources(dir string, files []string) []byte { imports := map[string]bool{} var bodies [][]byte pkgName := "" for _, f := range files { data, ok := readFile(joinPath(dir, f)) if !ok { continue } if pkgName == "" { pkgName = extractPkgName(data) } lines := splitLines(string(data)) var body []string i := int32(0) for i < int32(len(lines)) { line := trimSpace(lines[i]) if hasPrefix(line, "package ") { i++ continue } if line == "import (" { i++ for i < int32(len(lines)) { imp := trimSpace(lines[i]) i++ if imp == ")" { break } if imp != "" { imports[imp] = true } } continue } if hasPrefix(line, "import ") && !hasPrefix(line, "import (") { imp := line[7:] imports[imp] = true i++ continue } body = append(body, lines[i]) i++ } bodies = append(bodies, []byte(joinStrings(body, "\n"))) } var out []byte out = append(out, ("package " | pkgName | "\n")...) if len(imports) > 0 { var impKeys []string for imp := range imports { impKeys = append(impKeys, imp) } for i := int32(1); i < int32(len(impKeys)); i++ { for j := i; j > 0 && impKeys[j] < impKeys[j-1]; j-- { impKeys[j], impKeys[j-1] = impKeys[j-1], impKeys[j] } } out = append(out, "import (\n"...) for _, imp := range impKeys { out = append(out, '\t') out = append(out, imp...) out = append(out, '\n') } out = append(out, ")\n"...) } for _, b := range bodies { out = append(out, b...) out = append(out, '\n') } return out } func fileContainsPart(base string, part string) bool { plen := len(part) for i := 0; i <= len(base)-plen; i++ { if base[i:i+plen] == part { before := i == 0 || base[i-1] == '_' after := i+plen == len(base) || base[i+plen] == '_' if before && after { return true } } } return false } func shouldSkipFile(name string) bool { if hasSuffix(name, "_test.go") || hasSuffix(name, "_test.mx") { return true } base := name if hasSuffix(base, ".go") { base = base[:len(base)-3] } else if hasSuffix(base, ".mx") { base = base[:len(base)-3] } archSkip := []string{"arm64"} for _, a := range archSkip { if fileContainsPart(base, a) { return true } } suffixSkip := []string{"_wasm"} for _, s := range suffixSkip { if hasSuffix(base, s) { return true } } exact := []string{"pipe", "multi", "scan", "reader", "replace", "search", "ioutil"} for _, s := range exact { if base == s { return true } } return false } func hasBuildConstraint(data []byte, forbidden []string) bool { for i := int32(0); i < int32(len(data)) && i < 512; i++ { if data[i] == 'p' { return false } if i+11 < int32(len(data)) && string(data[i:i+11]) == "//go:build " { var buf []byte for j := i + 11; j < int32(len(data)) && data[j] != '\n'; j++ { buf = append(buf, data[j]) } line := string(buf) for _, f := range forbidden { if line == f || hasPrefix(line, f | " ") || hasPrefix(line, f | "\t") { return true } } return false } if data[i] == '\n' { continue } if data[i] == '/' && i+1 < int32(len(data)) && data[i+1] == '/' { for i < int32(len(data)) && data[i] != '\n' { i++ } continue } } return false } func hasImport(data []byte, pkg string) bool { target := "\"" | pkg | "\"" for i := int32(0); i <= int32(len(data))-int32(len(target)); i++ { if string(data[i:i+int32(len(target))]) == target { return true } } return false } type pkgInfo struct { path string name string dir string files []string cfiles []string bcfiles []string imports []string version string } func discoverPkg(pkgPath, root string) *pkgInfo { var dir string if pkgPath == "." || pkgPath == ".." || hasPrefix(pkgPath, "/") || hasPrefix(pkgPath, "./") || hasPrefix(pkgPath, "../") { dir = pkgPath } else if globalReplaces != nil { if rdir, ok := globalReplaces[pkgPath]; ok { dir = rdir } if dir == "" { for from, to := range globalReplaces { if hasPrefix(pkgPath, from|"/") { rel := pkgPath[len(from)+1:] dir = joinPath(to, rel) break } } } } if dir == "" && globalModPrefix != "" && pkgPath == globalModPrefix { dir = globalModDir } if dir == "" && globalModPrefix != "" && hasPrefix(pkgPath, globalModPrefix|"/") { rel := pkgPath[len(globalModPrefix)+1:] dir = joinPath(globalModDir, rel) } if dir == "" { candidate := joinPath(getMoxiePath(), pkgPath) mxF := listFiles(candidate, ".mx") if len(mxF) > 0 { dir = candidate } } if dir == "" && isRemotePath(pkgPath) { version := "" if globalModDir != "" { _, reqs, _ := parseModRequires(globalModDir) for _, r := range reqs { if r.path == pkgPath { version = r.version break } } } fetched := autoFetchRepo(pkgPath, version) if fetched != "" { mxF := listFiles(fetched, ".mx") if len(mxF) > 0 { dir = fetched } } } if dir != "" && dir != pkgPath { mergeModReplaces(dir) } if dir == "" { dir = joinPath(root, joinPath("src", pkgPath)) mxFiles := listFiles(dir, ".mx") if len(mxFiles) == 0 { dir = joinPath(root, joinPath("src/vendor", pkgPath)) } } goFiles := listFiles(dir, ".go") mxFiles := listFiles(dir, ".mx") cFiles := listFiles(dir, ".c") bcFiles := listFiles(dir, ".bc") allMx := append(mxFiles, goFiles...) forbidden := []string{"wasm", "js", "ignore", "compiler_bootstrap", "scheduler.tasks", "scheduler.cores", "scheduler.asyncify", "faketime", "baremetal", "nintendoswitch"} var files []string for _, f := range allMx { if shouldSkipFile(f) { continue } data, ok := readFile(joinPath(dir, f)) if !ok { continue } bc := hasBuildConstraint(data, forbidden) if bc { writeStr(2, " skip-constraint: " | f | "\n") continue } fileImps := extractImports(data) skipIter := false for _, imp := range fileImps { if imp == "iter" { skipIter = true break } } if skipIter { writeStr(2, " skip-iter-import: " | f | "\n") continue } writeStr(2, " include: " | f | "\n") files = append(files, f) } if len(files) == 0 { return nil } var allImports []string pkgName := "" for _, f := range files { data, ok := readFile(joinPath(dir, f)) if !ok { continue } if pkgName == "" { pkgName = extractPkgName(data) } for _, imp := range extractImports(data) { allImports = appendUniq(allImports, imp) } } writeStr(2, " discover " | pkgPath | " [") for fi, ff := range files { if fi > 0 { writeStr(2, ", ") } writeStr(2, ff) } writeStr(2, "]\n") version := "" if isRemotePath(pkgPath) && globalModDir != "" { _, reqs, _ := parseModRequires(globalModDir) for _, r := range reqs { if r.path == pkgPath || hasPrefix(pkgPath, r.path|"/") { version = r.version break } } } return &pkgInfo{ path: pkgPath, name: pkgName, dir: dir, files: files, cfiles: cFiles, bcfiles: bcFiles, imports: allImports, version: version, } } type resolver struct { visited map[string]bool order []*pkgInfo root string } func (r *resolver) walk(path string) { if r.visited[path] { return } r.visited[path] = true if isBuiltinOnly(path) { return } pkg := discoverPkg(path, r.root) if pkg == nil { return } for _, imp := range pkg.imports { r.walk(imp) } r.order = append(r.order, pkg) } func resolveAll(rootPkg string, root string) []*pkgInfo { r := &resolver{ visited: map[string]bool{}, root: root, } r.walk(rootPkg) return r.order } func registerBuiltins() { initUniverse() ensureImportRegistry() importRegistry = map[string]*TCPackage{} // unsafe - compiler builtin, no source unsafePkg := NewTCPackage("unsafe", "unsafe") unsafePkg.Scope().Insert(NewTypeName(unsafePkg, "Pointer", Typ[UnsafePointer])) unsafePkg.Scope().Insert(NewTCFunc(unsafePkg, "Sizeof", parseSignatureDesc("interface{}->int32"))) unsafePkg.Scope().Insert(NewTCFunc(unsafePkg, "Offsetof", parseSignatureDesc("interface{}->int32"))) unsafePkg.Scope().Insert(NewTCFunc(unsafePkg, "Alignof", parseSignatureDesc("interface{}->int32"))) unsafePkg.Scope().Insert(NewTCFunc(unsafePkg, "Slice", parseSignatureDesc("ptr,int32->[]uint8"))) unsafePkg.Scope().Insert(NewTCFunc(unsafePkg, "SliceData", parseSignatureDesc("[]uint8->ptr"))) importRegistry["unsafe"] = unsafePkg // moxie - spawn/codec runtime, special handling importRegistry["moxie"] = NewTCPackage("moxie", "moxie") // runtime - assembly + C, pre-compiled in runtime.bc rtPkg := NewTCPackage("runtime", "runtime") rtPkg.Scope().Insert(NewTCFunc(rtPkg, "InitCShared", parseSignatureDesc(""))) importRegistry["runtime"] = rtPkg } func addPtrMethod(pkg *TCPackage, named *Named, name, sigDesc string) { var sig *Signature if sigDesc == "" { sig = NewSignature(nil, nil, nil, false) } else { sig = parseSignatureDesc(sigDesc) } fn := NewTCFunc(pkg, name, sig) fn.hasPtrRecv = true named.AddMethod(fn) } func compileSource(src []byte, pkgName, triple string) string { ir := CompileToIR(src, pkgName, triple) if len(ir) > 0 && ir[0] == ';' { writeStr(2, ir) fatal("compile error for " | pkgName) } if len(ir) == 0 { fatal("empty IR for " | pkgName) } return ir } func typeDescStr(t Type) string { if t == nil { return "interface{}" } switch v := t.(type) { case *Basic: switch v.kind { case Bool: return "bool" case Int8: return "int8" case Int16: return "int16" case Int32: return "int32" case Int64: return "int64" case Uint8: return "uint8" case Uint16: return "uint16" case Uint32: return "uint32" case Uint64: return "uint64" case Float32: return "float32" case Float64: return "float64" case TCString: return "string" case UnsafePointer: return "ptr" } return "int32" case *Slice: return "[]" | typeDescStr(v.Elem()) case *Pointer: return "*" | typeDescStr(v.Elem()) case *TCInterface: if v.IsEmpty() { return "interface{}" } return "interface{}" case *Named: if v.obj != nil { return v.obj.name } return typeDescStr(v.underlying) } return "interface{}" } func sigDescStr(sig *Signature) string { var out string if sig.params != nil { for i := int32(0); i < sig.params.Len(); i++ { if i > 0 { out = out | "," } out = out | typeDescStr(sig.params.At(i).Type()) } } if sig.results != nil && sig.results.Len() > 0 { out = out | "->" for i := int32(0); i < sig.results.Len(); i++ { if i > 0 { out = out | "," } out = out | typeDescStr(sig.results.At(i).Type()) } } return out } func generateMxh(pkg *TCPackage) string { path := pkg.Path() name := pkg.Name() var out string out = "package " | path | " " | name | "\n" for _, objName := range pkg.Scope().Names() { if len(objName) == 0 || objName[0] < 'A' || objName[0] > 'Z' { continue } obj := pkg.Scope().Lookup(objName) if obj == nil { continue } switch v := obj.(type) { case *TCFunc: sig := v.Signature() if sig != nil { out = out | "func " | objName | " " | sigDescStr(sig) | "\n" } case *TCVar: out = out | "var " | objName | " " | typeDescStr(v.Type()) | "\n" case *TCConst: td := typeDescStr(v.Type()) vs := "" if v.val != nil { vs = v.val.String() } out = out | "const " | objName | " " | td | " " | vs | "\n" case *TypeName: typ := v.Type() if named, ok := typ.(*Named); ok { ud := "" if named.underlying != nil { ud = typeDescStr(named.underlying) } out = out | "type " | objName | " " | ud | "\n" for _, m := range named.methods { ptrRecv := "0" if m.hasPtrRecv { ptrRecv = "1" } msig := m.Signature() if msig != nil { out = out | "method " | objName | " " | m.Name() | " " | sigDescStr(msig) | " " | ptrRecv | "\n" } } } else if iface, ok := typ.(*TCInterface); ok { var methods string for mi := int32(0); mi < iface.NumMethods(); mi++ { m := iface.Method(mi) if mi > 0 { methods = methods | ";" } methods = methods | m.name | "=" | sigDescStr(m.sig) } out = out | "iface " | objName | " " | methods | "\n" } } } return out } func loadMxh(path string) bool { data, ok := readFile(path) if !ok { return false } ensureImportRegistry() lines := splitLines(string(data)) if len(lines) == 0 { return false } first := lines[0] if !hasPrefix(first, "package ") { return false } rest := first[8:] spaceIdx := int32(-1) for i := int32(0); i < int32(len(rest)); i++ { if rest[i] == ' ' { spaceIdx = i break } } if spaceIdx < 0 { return false } pkgPath := rest[:spaceIdx] pkgName := rest[spaceIdx+1:] pkg := NewTCPackage(pkgPath, pkgName) for li := 1; li < len(lines); li++ { line := lines[li] if hasPrefix(line, "func ") { parts := splitBySpace(line[5:]) if len(parts) >= 2 { sig := parseSignatureDesc(parts[1]) pkg.Scope().Insert(NewTCFunc(pkg, parts[0], sig)) } else if len(parts) == 1 { sig := parseSignatureDesc("") pkg.Scope().Insert(NewTCFunc(pkg, parts[0], sig)) } } else if hasPrefix(line, "var ") { parts := splitBySpace(line[4:]) if len(parts) >= 2 { typ := parseTypeDesc(parts[1]) pkg.Scope().Insert(NewTCVar(pkg, parts[0], typ)) } } else if hasPrefix(line, "const ") { parts := splitBySpace(line[6:]) if len(parts) >= 2 { typ := parseTypeDesc(parts[1]) var val ConstVal if len(parts) >= 3 { val = constStr{parts[2]} } else { val = constInt{0} } pkg.Scope().Insert(NewTCConst(pkg, parts[0], typ, val)) } } else if hasPrefix(line, "type ") { parts := splitBySpace(line[5:]) if len(parts) >= 2 { underlying := parseTypeDesc(parts[1]) tn := NewTypeName(pkg, parts[0], nil) NewNamed(tn, underlying) pkg.Scope().Insert(tn) } else if len(parts) == 1 { underlying := NewTCStruct(nil, nil) tn := NewTypeName(pkg, parts[0], nil) NewNamed(tn, underlying) pkg.Scope().Insert(tn) } } else if hasPrefix(line, "method ") { parts := splitBySpace(line[7:]) if len(parts) >= 3 { typeName := parts[0] methodName := parts[1] sigDesc := parts[2] ptrRecv := false if len(parts) >= 4 && parts[3] == "1" { ptrRecv = true } obj := pkg.Scope().Lookup(typeName) if obj != nil { if tn, ok := obj.(*TypeName); ok { if named, ok := tn.typ.(*Named); ok { sig := parseSignatureDesc(sigDesc) fn := NewTCFunc(pkg, methodName, sig) fn.hasPtrRecv = ptrRecv named.AddMethod(fn) } } } } } else if hasPrefix(line, "iface ") { parts := splitBySpace(line[6:]) if len(parts) >= 1 { ifaceName := parts[0] methodsDesc := "" if len(parts) >= 2 { methodsDesc = parts[1] } var methods []*IfaceMethod if methodsDesc != "" { mparts := splitSemicolon(methodsDesc) for _, p := range mparts { eqIdx := -1 for i := 0; i < len(p); i++ { if p[i] == '=' { eqIdx = i break } } if eqIdx < 0 { continue } mname := p[:eqIdx] msig := parseSignatureDesc(p[eqIdx+1:]) methods = append(methods, NewTCIfaceMethod(mname, msig)) } } iface := NewTCInterface(methods, nil) iface.Complete() tn := NewTypeName(pkg, ifaceName, iface) pkg.Scope().Insert(tn) } } } importRegistry[pkgPath] = pkg return true } func cachedMxhPath(cacheBase, pkgPath, hash, version string) string { return joinPath(joinPath(cacheBase, safeName(pkgPath)), versionedCacheName(hash, version, ".mxh")) } func removeAll(path string) { entries := listFiles(path, "") if entries == nil { return } for _, e := range entries { os.Remove(joinPath(path, e)) } os.Remove(path) } func cacheClean() { base := pkgCacheDir() writeStr(2, "cleaning cache: " | base | "\n") entries := listFiles(base, "") if entries == nil { writeStr(2, " (empty)\n") return } for _, e := range entries { removeAll(joinPath(base, e)) } os.Remove(base) writeStr(2, " done\n") } func cmdUpdate(rebuildAll bool) { cacheRoot := joinPath(getMoxiePath(), "cache") currentDir := "mxc-" | mxcVersion() entries := listFiles(cacheRoot, "") if entries == nil { writeStr(2, "cache is empty\n") return } for _, e := range entries { if hasPrefix(e, "mxc-") && e != currentDir { writeStr(2, " remove old: " | e | "\n") fullPath := joinPath(cacheRoot, e) pkgEntries := listFiles(fullPath, "") if pkgEntries != nil { for _, pe := range pkgEntries { removeAll(joinPath(fullPath, pe)) } } os.Remove(fullPath) } } if !rebuildAll { writeStr(2, "update done\n") return } writeStr(2, "rebuilding all cached packages...\n") mpath := getMoxiePath() root := getenv("MOXIEROOT") if root == "" { root = "." } cacheBase := pkgCacheDir() pkgDirs := listFiles(cacheBase, "") if pkgDirs == nil { writeStr(2, " no packages to rebuild\n") return } triple := "x86_64-unknown-linux-musleabihf" clang := "clang-21" tmpdir := "/tmp/mxc-build" mkdirAll(tmpdir, 0755) registerBuiltins() for _, pkgDir := range pkgDirs { pkgPath := "" for i := 0; i < len(pkgDir); i++ { if pkgDir[i] == '_' { pkgPath = pkgPath | "/" } else { pkgPath = pkgPath | string([]byte{pkgDir[i]}) } } writeStr(2, " rebuild " | pkgPath | "\n") pkg := discoverPkg(pkgPath, root) if pkg == nil { candidate := joinPath(mpath, pkgPath) mxF := listFiles(candidate, ".mx") if len(mxF) > 0 { pkg = discoverPkg(pkgPath, root) } } if pkg == nil { writeStr(2, " skip (source not found)\n") continue } compilePath := pkg.path if hasPrefix(compilePath, "/") || hasPrefix(compilePath, "./") || hasPrefix(compilePath, "../") { compilePath = pkg.name } src := concatSources(pkg.dir, pkg.files) ir := compileSource(src, compilePath, triple) src = nil llFile := joinPath(tmpdir, safeName(pkg.path) | ".ll") writeFile(llFile, []byte(ir), 0644) ir = "" bcFile := joinPath(tmpdir, safeName(pkg.path) | ".bc") rc := run([]string{clang, "--target=" | triple, "-emit-llvm", "-c", llFile, "-o", bcFile}) if rc != 0 { writeStr(2, " FAIL: clang for " | pkg.path | "\n") continue } hash := pkgCacheKey(pkg.dir, pkg.files) cached := cachedBcPath(cacheBase, pkg.path, hash, pkg.version) cdir := joinPath(cacheBase, safeName(pkg.path)) mkdirAll(cdir, 0755) data, rok := readFile(bcFile) if rok { writeFile(cached, data, 0644) } data = nil regPkg := importRegistry[compilePath] if regPkg != nil { cachedMxh := cachedMxhPath(cacheBase, pkg.path, hash, pkg.version) mxhData := generateMxh(regPkg) writeFile(cachedMxh, []byte(mxhData), 0644) } } writeStr(2, "rebuild done\n") } func main() { builtinOnly = []string{"unsafe", "runtime", "moxie"} args := getArgs() if len(args) < 2 { writeStr(2, "usage: mxc ...\n") os.Exit(1) } cmd := args[1] if cmd == "version" { writeStr(1, "mxc " | mxcVersion() | "\n") return } if cmd == "cache" { if len(args) >= 3 && args[2] == "clean" { cacheClean() return } writeStr(2, "usage: mxc cache clean\n") os.Exit(1) } if cmd == "fetch" { cmdFetch() return } if cmd == "update" { rebuildAll := false for ai := int32(2); ai < int32(len(args)); ai++ { if args[ai] == "--all" { rebuildAll = true } } cmdUpdate(rebuildAll) return } if cmd != "build" { writeStr(2, "usage: mxc ...\n") os.Exit(1) } outpath := "" pkgPath := "" printIR := false forceRebuild := false i := int32(2) for i < int32(len(args)) { if args[i] == "-o" && i+1 < int32(len(args)) { outpath = args[i+1] i += 2 } else if args[i] == "-print-ir" { printIR = true i++ } else if args[i] == "-a" { forceRebuild = true i++ } else { pkgPath = args[i] i++ } } if pkgPath == "" { fatal("no package specified") } if outpath == "" { outpath = pathBase(pkgPath) } root := getenv("MOXIEROOT") if root == "" { root = "." } triple := "x86_64-unknown-linux-musleabihf" clang := "clang-21" if hasPrefix(pkgPath, "./") || hasPrefix(pkgPath, "../") || pkgPath == "." { absDir := pkgPath if absDir == "." { absDir = "" } globalModPrefix, globalModDir = findModuleRoot(absDir) } registerBuiltins() pkgs := resolveAll(pkgPath, root) if len(pkgs) == 0 { fatal("no packages to compile") } tmpdir := "/tmp/mxc-build" mkdirAll(tmpdir, 0755) cacheBase := pkgCacheDir() var allBcFiles []string for _, pkg := range pkgs { compilePath := pkg.path if hasPrefix(compilePath, "/") || hasPrefix(compilePath, "./") || hasPrefix(compilePath, "../") { compilePath = pkg.name } hash := pkgCacheKey(pkg.dir, pkg.files) cached := cachedBcPath(cacheBase, pkg.path, hash, pkg.version) cachedMxh := cachedMxhPath(cacheBase, pkg.path, hash, pkg.version) _, bcOk := readFile(cached) _, mxhOk := readFile(cachedMxh) if bcOk && mxhOk && !forceRebuild { writeStr(2, " cached " | pkg.path) if pkg.version != "" { writeStr(2, " @ " | pkg.version) } writeStr(2, "\n") loadMxh(cachedMxh) allBcFiles = append(allBcFiles, cached) for _, bf := range pkg.bcfiles { allBcFiles = append(allBcFiles, joinPath(pkg.dir, bf)) } continue } if isStdlib(pkg.path) && bcOk && !forceRebuild { writeStr(2, " cached " | pkg.path | "\n") src := concatSources(pkg.dir, pkg.files) TypeCheckOnly(src, compilePath) src = nil regPkg := importRegistry[compilePath] if regPkg != nil { cacheDir := joinPath(cacheBase, safeName(pkg.path)) mkdirAll(cacheDir, 0755) mxhData := generateMxh(regPkg) writeFile(cachedMxh, []byte(mxhData), 0644) } allBcFiles = append(allBcFiles, cached) continue } writeStr(2, " compile " | pkg.path | "\n") src := concatSources(pkg.dir, pkg.files) ir := compileSource(src, compilePath, triple) src = nil if printIR { writeStr(1, ir) } llFile := joinPath(tmpdir, safeName(pkg.path) | ".ll") writeFile(llFile, []byte(ir), 0644) ir = "" bcFile := joinPath(tmpdir, safeName(pkg.path) | ".bc") rc := run([]string{clang, "--target=" | triple, "-emit-llvm", "-c", llFile, "-o", bcFile}) if rc != 0 { fatal("clang failed for " | pkg.path) } cacheDir := joinPath(cacheBase, safeName(pkg.path)) mkdirAll(cacheDir, 0755) data, rok := readFile(bcFile) if rok { writeFile(cached, data, 0644) } data = nil regPkg := importRegistry[compilePath] if regPkg != nil { mxhData := generateMxh(regPkg) writeFile(cachedMxh, []byte(mxhData), 0644) } allBcFiles = append(allBcFiles, bcFile) for _, cf := range pkg.cfiles { cPath := joinPath(pkg.dir, cf) cbcFile := joinPath(tmpdir, safeName(pkg.path) | "_" | cf | ".bc") rc = run([]string{clang, "--target=" | triple, "-emit-llvm", "-c", cPath, "-o", cbcFile}) if rc != 0 { fatal("clang failed for " | cf) } allBcFiles = append(allBcFiles, cbcFile) } for _, bf := range pkg.bcfiles { allBcFiles = append(allBcFiles, joinPath(pkg.dir, bf)) } } loadSysroot(joinPath(root, "_sysroot.env")) sysroot := joinPath(root, "sysroot") runtimeBc := getenv("RUNTIME_BC") if runtimeBc == "" { runtimeBc = joinPath(sysroot, "runtime.bc") _, rok := readFile(runtimeBc) if !rok { runtimeBc = joinPath(root, "_runtime.bc") } } muslDir := getenv("MUSL_DIR") if muslDir == "" { muslDir = joinPath(sysroot, "musl") } comprtLib := getenv("COMPRT_LIB") if comprtLib == "" { comprtLib = joinPath(joinPath(sysroot, "compiler-rt"), "lib.a") } bdwgcLib := getenv("BDWGC_LIB") if bdwgcLib == "" { bdwgcLib = joinPath(joinPath(sysroot, "bdwgc"), "lib.a") } muslLib := getenv("MUSL_LIB") if muslLib == "" { muslLib = joinPath(joinPath(sysroot, "musl"), "lib.a") } writeStr(2, " generate initAll\n") var initCalls string for _, pkg := range pkgs { compilePath := pkg.path if hasPrefix(compilePath, "/") || hasPrefix(compilePath, "./") || hasPrefix(compilePath, "../") { compilePath = pkg.name } if compilePath == "main" { continue } initSym := irGlobalSymbol(compilePath, "main") initCalls = initCalls | " call void " | initSym | "(ptr %context)\n" } initAllIR := "\ndefine void @runtime.initAll(ptr %context) {\nentry:\n" | initCalls | " ret void\n}\n" declared := map[string]bool{} for _, pkg := range pkgs { compilePath := pkg.path if hasPrefix(compilePath, "/") || hasPrefix(compilePath, "./") || hasPrefix(compilePath, "../") { compilePath = pkg.name } if compilePath == "main" { continue } declSym := irGlobalSymbol(compilePath, "main") if declared[declSym] { continue } declared[declSym] = true initAllIR = initAllIR | "declare void " | declSym | "(ptr)\n" } initAllFile := joinPath(tmpdir, "initall.ll") writeFile(initAllFile, []byte(initAllIR), 0644) initAllBc := joinPath(tmpdir, "initall.bc") rc := run([]string{clang, "--target=" | triple, "-emit-llvm", "-c", initAllFile, "-o", initAllBc}) if rc != 0 { fatal("clang failed for initall") } writeStr(2, " merge bitcode\n") llvmLink := "llvm-link-21" mergedBc := joinPath(tmpdir, "merged.bc") linkArgs := []string{llvmLink, "-o", mergedBc, runtimeBc} shimsBc := joinPath(sysroot, "hashmap_shims.bc") linkArgs = append(linkArgs, shimsBc) runtimeGoBc := joinPath(sysroot, "runtime_go.bc") if _, gok := readFile(runtimeGoBc); gok { overrideArg := "--override=" | runtimeGoBc linkArgs = append(linkArgs, overrideArg) } linkArgs = append(linkArgs, allBcFiles...) linkArgs = append(linkArgs, initAllBc) rc = run(linkArgs) if rc != 0 { fatal("llvm-link failed") } writeStr(2, " link\n") objFiles := getenv("OBJ_FILES") if objFiles == "" { objEntries := listFiles(joinPath(sysroot, "obj"), ".bc") if objEntries != nil { for _, e := range objEntries { objFiles = objFiles | joinPath(joinPath(sysroot, "obj"), e) | " " } } } linker := "ld.lld-21" ldArgs := []string{linker, "--gc-sections", "-o", outpath, joinPath(muslDir, "crt1.o"), mergedBc} for _, f := range splitBySpace(objFiles) { if f != "" { ldArgs = append(ldArgs, f) } } ldArgs = append(ldArgs, comprtLib, bdwgcLib, muslLib, "--lto-O0", "-z", "stack-size=67108864") rc = run(ldArgs) if rc != 0 { fatal("linker failed") } writeStr(2, " -> " | outpath | "\n") }