package loader // This file constructs a new temporary GOROOT directory by merging both the // standard Go GOROOT and the GOROOT from Moxie using symlinks. // // The goal is to replace specific packages from Go with a Moxie version. It's // never a partial replacement, either a package is fully replaced or it is not. // This is important because if we did allow to merge packages (e.g. by adding // files to a package), it would lead to a dependency on implementation details // with all the maintenance burden that results in. Only allowing to replace // packages as a whole avoids this as packages are already designed to have a // public (backwards-compatible) API. import ( "crypto/sha512" "encoding/hex" "encoding/json" "errors" "io" "io/fs" "os" "os/exec" "path" "path/filepath" "runtime" "sort" "sync" "moxie/compileopts" "moxie/goenv" ) var gorootCreateMutex sync.Mutex // GetCachedGoroot creates a new GOROOT by merging both the standard GOROOT and // the GOROOT from Moxie using lots of symbolic links. func GetCachedGoroot(config *compileopts.Config) (string, error) { goroot := goenv.Get("GOROOT") if goroot == "" { return "", errors.New("could not determine GOROOT") } moxieroot := goenv.Get("MOXIEROOT") if moxieroot == "" { return "", errors.New("could not determine MOXIEROOT") } // Find the overrides needed for the goroot. overrides := pathsToOverride(config.GoMinorVersion, needsSyscallPackage(config.BuildTags())) // Resolve the merge links within the goroot. merge, err := listGorootMergeLinks(goroot, moxieroot, overrides) if err != nil { return "", err } // Hash the merge links to create a cache key. data, err := json.Marshal(merge) if err != nil { return "", err } hash := sha512.Sum512_256(data) // Do not try to create the cached GOROOT in parallel, that's only a waste // of I/O bandwidth and thus speed. Instead, use a mutex to make sure only // one goroutine does it at a time. // This is not a way to ensure atomicity (a different Moxie invocation // could be creating the same directory), but instead a way to avoid // creating it many times in parallel when running tests in parallel. gorootCreateMutex.Lock() defer gorootCreateMutex.Unlock() // Check if the goroot already exists. cachedGorootName := "goroot-" + hex.EncodeToString(hash[:]) cachedgoroot := filepath.Join(goenv.Get("GOCACHE"), cachedGorootName) if _, err := os.Stat(cachedgoroot); err == nil { return cachedgoroot, nil } // Create the cache directory if it does not already exist. err = os.MkdirAll(goenv.Get("GOCACHE"), 0777) if err != nil { return "", err } // Create a temporary directory to construct the goroot within. tmpgoroot, err := os.MkdirTemp(goenv.Get("GOCACHE"), cachedGorootName+".tmp") if err != nil { return "", err } // Remove the temporary directory if it wasn't moved to the right place // (for example, when there was an error). defer os.RemoveAll(tmpgoroot) // Create the directory structure. // The directories are created in sorted order so that nested directories are created without extra work. { var dirs []string for dir, merge := range overrides { if merge { dirs = append(dirs, filepath.Join(tmpgoroot, "src", dir)) } } sort.Strings(dirs) for _, dir := range dirs { err := os.Mkdir(dir, 0777) if err != nil { return "", err } } } // Create all symlinks. for dst, src := range merge { err := symlink(src, filepath.Join(tmpgoroot, dst)) if err != nil { return "", err } } // Rename the new merged gorooot into place. err = os.Rename(tmpgoroot, cachedgoroot) if err != nil { if errors.Is(err, fs.ErrExist) { // Another invocation of Moxie also seems to have created a GOROOT. // Use that one instead. Our new GOROOT will be automatically // deleted by the defer above. return cachedgoroot, nil } if runtime.GOOS == "windows" && errors.Is(err, fs.ErrPermission) { // On Windows, a rename with a destination directory that already // exists does not result in an IsExist error, but rather in an // access denied error. To be sure, check for this case by checking // whether the target directory exists. if _, err := os.Stat(cachedgoroot); err == nil { return cachedgoroot, nil } } return "", err } return cachedgoroot, nil } // listGorootMergeLinks searches goroot and moxieroot for all symlinks that must be created within the merged goroot. func listGorootMergeLinks(goroot, moxieroot string, overrides map[string]bool) (map[string]string, error) { goSrc := filepath.Join(goroot, "src") moxieSrc := filepath.Join(moxieroot, "src") merges := make(map[string]string) for dir, merge := range overrides { if !merge { // Use the Moxie version. merges[filepath.Join("src", dir)] = filepath.Join(moxieSrc, dir) continue } // Add files from Moxie. moxieDir := filepath.Join(moxieSrc, dir) moxieEntries, err := os.ReadDir(moxieDir) if err != nil { return nil, err } var hasMoxieFiles bool for _, e := range moxieEntries { if e.IsDir() { continue } // Link this file. name := e.Name() merges[filepath.Join("src", dir, name)] = filepath.Join(moxieDir, name) hasMoxieFiles = true } // Add all directories from $GOROOT that are not part of the Moxie // overrides. goDir := filepath.Join(goSrc, dir) goEntries, err := os.ReadDir(goDir) if err != nil { return nil, err } for _, e := range goEntries { isDir := e.IsDir() if hasMoxieFiles && !isDir { // Only merge files from Go if Moxie does not have any files. // Otherwise we'd end up with a weird mix from both Go // implementations. continue } name := e.Name() if _, ok := overrides[path.Join(dir, name)+"/"]; ok { // This entry is overridden by Moxie. // It has/will be merged elsewhere. continue } // Add a link to this entry merges[filepath.Join("src", dir, name)] = filepath.Join(goDir, name) } } // Merge the special directories from goroot. for _, dir := range []string{"bin", "lib", "pkg"} { merges[dir] = filepath.Join(goroot, dir) } // Required starting in Go 1.21 due to https://github.com/golang/go/issues/61928 if _, err := os.Stat(filepath.Join(goroot, "go.env")); err == nil { merges["go.env"] = filepath.Join(goroot, "go.env") } return merges, nil } // needsSyscallPackage returns whether the syscall package should be overridden // with the Moxie version. This is the case on some targets. func needsSyscallPackage(buildTags []string) bool { for _, tag := range buildTags { if tag == "baremetal" || tag == "nintendoswitch" || tag == "moxie.wasm" { return true } } return false } // The boolean indicates whether to merge the subdirs. True means merge, false // means use the moxie version entirely. // // Moxie's src/ tree is self-contained — all stdlib source is in-tree. // The root entry (false) means use moxie's src/ for everything. // syscall/ is the one exception: moxie has compiler-intrinsic stubs that // conflict with Go's full implementation. The merge system provides Go's // syscall files for hosted targets while moxie's stubs serve baremetal. func pathsToOverride(goMinor int, needsSyscallPackage bool) map[string]bool { return map[string]bool{ "": false, } } // symlink creates a symlink or something similar. On Unix-like systems, it // always creates a symlink. On Windows, it tries to create a symlink and if // that fails, creates a hardlink or directory junction instead. // // Note that while Windows 10 does support symlinks and allows them to be // created using os.Symlink, it requires developer mode to be enabled. // Therefore provide a fallback for when symlinking is not possible. // Unfortunately this fallback only works when Moxie is installed on the same // filesystem as the Moxie cache and the Go installation (which is usually the // C drive). func symlink(oldname, newname string) error { symlinkErr := os.Symlink(oldname, newname) if runtime.GOOS == "windows" && symlinkErr != nil { // Fallback for when developer mode is disabled. // Note that we return the symlink error even if something else fails // later on. This is because symlinks are the easiest to support // (they're also used on Linux and MacOS) and enabling them is easy: // just enable developer mode. st, err := os.Stat(oldname) if err != nil { return symlinkErr } if st.IsDir() { // Make a directory junction. There may be a way to do this // programmatically, but it involves a lot of magic. Use the mklink // command built into cmd instead (mklink is a builtin, not an // external command). err := exec.Command("cmd", "/k", "mklink", "/J", newname, oldname).Run() if err != nil { return symlinkErr } } else { // Try making a hard link. err := os.Link(oldname, newname) if err != nil { // Making a hardlink failed. Try copying the file as a last // fallback. inf, err := os.Open(oldname) if err != nil { return err } defer inf.Close() outf, err := os.Create(newname) if err != nil { return err } defer outf.Close() _, err = io.Copy(outf, inf) if err != nil { os.Remove(newname) return err } // File was copied. } } return nil // success } return symlinkErr }