1 package loader
2 3 // This file constructs a new temporary GOROOT directory by merging both the
4 // standard Go GOROOT and the GOROOT from Moxie using symlinks.
5 //
6 // The goal is to replace specific packages from Go with a Moxie version. It's
7 // never a partial replacement, either a package is fully replaced or it is not.
8 // This is important because if we did allow to merge packages (e.g. by adding
9 // files to a package), it would lead to a dependency on implementation details
10 // with all the maintenance burden that results in. Only allowing to replace
11 // packages as a whole avoids this as packages are already designed to have a
12 // public (backwards-compatible) API.
13 14 import (
15 "crypto/sha512"
16 "encoding/hex"
17 "encoding/json"
18 "errors"
19 "io"
20 "io/fs"
21 "os"
22 "os/exec"
23 "path"
24 "path/filepath"
25 "runtime"
26 "sort"
27 "sync"
28 29 "moxie/compileopts"
30 "moxie/goenv"
31 )
32 33 var gorootCreateMutex sync.Mutex
34 35 // GetCachedGoroot creates a new GOROOT by merging both the standard GOROOT and
36 // the GOROOT from Moxie using lots of symbolic links.
37 func GetCachedGoroot(config *compileopts.Config) (string, error) {
38 goroot := goenv.Get("GOROOT")
39 if goroot == "" {
40 return "", errors.New("could not determine GOROOT")
41 }
42 moxieroot := goenv.Get("MOXIEROOT")
43 if moxieroot == "" {
44 return "", errors.New("could not determine MOXIEROOT")
45 }
46 47 // Find the overrides needed for the goroot.
48 overrides := pathsToOverride(config.GoMinorVersion, needsSyscallPackage(config.BuildTags()))
49 50 // Resolve the merge links within the goroot.
51 merge, err := listGorootMergeLinks(goroot, moxieroot, overrides)
52 if err != nil {
53 return "", err
54 }
55 56 // Hash the merge links to create a cache key.
57 data, err := json.Marshal(merge)
58 if err != nil {
59 return "", err
60 }
61 hash := sha512.Sum512_256(data)
62 63 // Do not try to create the cached GOROOT in parallel, that's only a waste
64 // of I/O bandwidth and thus speed. Instead, use a mutex to make sure only
65 // one goroutine does it at a time.
66 // This is not a way to ensure atomicity (a different Moxie invocation
67 // could be creating the same directory), but instead a way to avoid
68 // creating it many times in parallel when running tests in parallel.
69 gorootCreateMutex.Lock()
70 defer gorootCreateMutex.Unlock()
71 72 // Check if the goroot already exists.
73 cachedGorootName := "goroot-" + hex.EncodeToString(hash[:])
74 cachedgoroot := filepath.Join(goenv.Get("GOCACHE"), cachedGorootName)
75 if _, err := os.Stat(cachedgoroot); err == nil {
76 return cachedgoroot, nil
77 }
78 79 // Create the cache directory if it does not already exist.
80 err = os.MkdirAll(goenv.Get("GOCACHE"), 0777)
81 if err != nil {
82 return "", err
83 }
84 85 // Create a temporary directory to construct the goroot within.
86 tmpgoroot, err := os.MkdirTemp(goenv.Get("GOCACHE"), cachedGorootName+".tmp")
87 if err != nil {
88 return "", err
89 }
90 91 // Remove the temporary directory if it wasn't moved to the right place
92 // (for example, when there was an error).
93 defer os.RemoveAll(tmpgoroot)
94 95 // Create the directory structure.
96 // The directories are created in sorted order so that nested directories are created without extra work.
97 {
98 var dirs []string
99 for dir, merge := range overrides {
100 if merge {
101 dirs = append(dirs, filepath.Join(tmpgoroot, "src", dir))
102 }
103 }
104 sort.Strings(dirs)
105 106 for _, dir := range dirs {
107 err := os.Mkdir(dir, 0777)
108 if err != nil {
109 return "", err
110 }
111 }
112 }
113 114 // Create all symlinks.
115 for dst, src := range merge {
116 err := symlink(src, filepath.Join(tmpgoroot, dst))
117 if err != nil {
118 return "", err
119 }
120 }
121 122 // Rename the new merged gorooot into place.
123 err = os.Rename(tmpgoroot, cachedgoroot)
124 if err != nil {
125 if errors.Is(err, fs.ErrExist) {
126 // Another invocation of Moxie also seems to have created a GOROOT.
127 // Use that one instead. Our new GOROOT will be automatically
128 // deleted by the defer above.
129 return cachedgoroot, nil
130 }
131 if runtime.GOOS == "windows" && errors.Is(err, fs.ErrPermission) {
132 // On Windows, a rename with a destination directory that already
133 // exists does not result in an IsExist error, but rather in an
134 // access denied error. To be sure, check for this case by checking
135 // whether the target directory exists.
136 if _, err := os.Stat(cachedgoroot); err == nil {
137 return cachedgoroot, nil
138 }
139 }
140 return "", err
141 }
142 return cachedgoroot, nil
143 }
144 145 // listGorootMergeLinks searches goroot and moxieroot for all symlinks that must be created within the merged goroot.
146 func listGorootMergeLinks(goroot, moxieroot string, overrides map[string]bool) (map[string]string, error) {
147 goSrc := filepath.Join(goroot, "src")
148 moxieSrc := filepath.Join(moxieroot, "src")
149 merges := make(map[string]string)
150 for dir, merge := range overrides {
151 if !merge {
152 // Use the Moxie version.
153 merges[filepath.Join("src", dir)] = filepath.Join(moxieSrc, dir)
154 continue
155 }
156 157 // Add files from Moxie.
158 moxieDir := filepath.Join(moxieSrc, dir)
159 moxieEntries, err := os.ReadDir(moxieDir)
160 if err != nil {
161 return nil, err
162 }
163 var hasMoxieFiles bool
164 for _, e := range moxieEntries {
165 if e.IsDir() {
166 continue
167 }
168 169 // Link this file.
170 name := e.Name()
171 merges[filepath.Join("src", dir, name)] = filepath.Join(moxieDir, name)
172 173 hasMoxieFiles = true
174 }
175 176 // Add all directories from $GOROOT that are not part of the Moxie
177 // overrides.
178 goDir := filepath.Join(goSrc, dir)
179 goEntries, err := os.ReadDir(goDir)
180 if err != nil {
181 return nil, err
182 }
183 for _, e := range goEntries {
184 isDir := e.IsDir()
185 if hasMoxieFiles && !isDir {
186 // Only merge files from Go if Moxie does not have any files.
187 // Otherwise we'd end up with a weird mix from both Go
188 // implementations.
189 continue
190 }
191 192 name := e.Name()
193 if _, ok := overrides[path.Join(dir, name)+"/"]; ok {
194 // This entry is overridden by Moxie.
195 // It has/will be merged elsewhere.
196 continue
197 }
198 199 // Add a link to this entry
200 merges[filepath.Join("src", dir, name)] = filepath.Join(goDir, name)
201 }
202 }
203 204 // Merge the special directories from goroot.
205 for _, dir := range []string{"bin", "lib", "pkg"} {
206 merges[dir] = filepath.Join(goroot, dir)
207 }
208 209 // Required starting in Go 1.21 due to https://github.com/golang/go/issues/61928
210 if _, err := os.Stat(filepath.Join(goroot, "go.env")); err == nil {
211 merges["go.env"] = filepath.Join(goroot, "go.env")
212 }
213 214 return merges, nil
215 }
216 217 // needsSyscallPackage returns whether the syscall package should be overridden
218 // with the Moxie version. This is the case on some targets.
219 func needsSyscallPackage(buildTags []string) bool {
220 for _, tag := range buildTags {
221 if tag == "baremetal" || tag == "nintendoswitch" || tag == "moxie.wasm" {
222 return true
223 }
224 }
225 return false
226 }
227 228 // The boolean indicates whether to merge the subdirs. True means merge, false
229 // means use the moxie version entirely.
230 //
231 // Moxie's src/ tree is self-contained — all stdlib source is in-tree.
232 // The root entry (false) means use moxie's src/ for everything.
233 // syscall/ is the one exception: moxie has compiler-intrinsic stubs that
234 // conflict with Go's full implementation. The merge system provides Go's
235 // syscall files for hosted targets while moxie's stubs serve baremetal.
236 func pathsToOverride(goMinor int, needsSyscallPackage bool) map[string]bool {
237 return map[string]bool{
238 "": false,
239 }
240 }
241 242 // symlink creates a symlink or something similar. On Unix-like systems, it
243 // always creates a symlink. On Windows, it tries to create a symlink and if
244 // that fails, creates a hardlink or directory junction instead.
245 //
246 // Note that while Windows 10 does support symlinks and allows them to be
247 // created using os.Symlink, it requires developer mode to be enabled.
248 // Therefore provide a fallback for when symlinking is not possible.
249 // Unfortunately this fallback only works when Moxie is installed on the same
250 // filesystem as the Moxie cache and the Go installation (which is usually the
251 // C drive).
252 func symlink(oldname, newname string) error {
253 symlinkErr := os.Symlink(oldname, newname)
254 if runtime.GOOS == "windows" && symlinkErr != nil {
255 // Fallback for when developer mode is disabled.
256 // Note that we return the symlink error even if something else fails
257 // later on. This is because symlinks are the easiest to support
258 // (they're also used on Linux and MacOS) and enabling them is easy:
259 // just enable developer mode.
260 st, err := os.Stat(oldname)
261 if err != nil {
262 return symlinkErr
263 }
264 if st.IsDir() {
265 // Make a directory junction. There may be a way to do this
266 // programmatically, but it involves a lot of magic. Use the mklink
267 // command built into cmd instead (mklink is a builtin, not an
268 // external command).
269 err := exec.Command("cmd", "/k", "mklink", "/J", newname, oldname).Run()
270 if err != nil {
271 return symlinkErr
272 }
273 } else {
274 // Try making a hard link.
275 err := os.Link(oldname, newname)
276 if err != nil {
277 // Making a hardlink failed. Try copying the file as a last
278 // fallback.
279 inf, err := os.Open(oldname)
280 if err != nil {
281 return err
282 }
283 defer inf.Close()
284 outf, err := os.Create(newname)
285 if err != nil {
286 return err
287 }
288 defer outf.Close()
289 _, err = io.Copy(outf, inf)
290 if err != nil {
291 os.Remove(newname)
292 return err
293 }
294 // File was copied.
295 }
296 }
297 return nil // success
298 }
299 return symlinkErr
300 }
301