1 package builder
2 3 // This file implements a wrapper around the C compiler (Clang) which uses a
4 // build cache.
5 6 import (
7 "crypto/sha512"
8 "encoding/hex"
9 "encoding/json"
10 "errors"
11 "fmt"
12 "io"
13 "io/fs"
14 "os"
15 "path/filepath"
16 "sort"
17 "strings"
18 "unicode"
19 20 "moxie/goenv"
21 "tinygo.org/x/go-llvm"
22 )
23 24 // compileAndCacheCFile compiles a C or assembly file using a build cache.
25 // Compiling the same file again (if nothing changed, including included header
26 // files) the output is loaded from the build cache instead.
27 //
28 // Its operation is a bit complex (more complex than Go package build caching)
29 // because the list of file dependencies is only known after the file is
30 // compiled. However, luckily compilers have a flag to write a list of file
31 // dependencies in Makefile syntax which can be used for caching.
32 //
33 // Because of this complexity, every file has in fact two cached build outputs:
34 // the file itself, and the list of dependencies. Its operation is as follows:
35 //
36 // depfile = hash(path, compiler, cflags, ...)
37 // if depfile exists:
38 // outfile = hash of all files and depfile name
39 // if outfile exists:
40 // # cache hit
41 // return outfile
42 // # cache miss
43 // tmpfile = compile file
44 // read dependencies (side effect of compile)
45 // write depfile
46 // outfile = hash of all files and depfile name
47 // rename tmpfile to outfile
48 //
49 // There are a few edge cases that are not handled:
50 // - If a file is added to an include path, that file may be included instead of
51 // some other file. This would be fixed by also including lookup failures in the
52 // dependencies file, but I'm not aware of a compiler which does that.
53 // - The Makefile syntax that compilers output has issues, see readDepFile for
54 // details.
55 // - A header file may be changed to add/remove an include. This invalidates the
56 // depfile but without invalidating its name. For this reason, the depfile is
57 // written on each new compilation (even when it seems unnecessary). However, it
58 // could in rare cases lead to a stale file fetched from the cache.
59 func compileAndCacheCFile(abspath, tmpdir string, cflags []string, printCommands func(string, ...string)) (string, error) {
60 // Hash input file.
61 fileHash, err := hashFile(abspath)
62 if err != nil {
63 return "", err
64 }
65 66 // Acquire a lock (if supported).
67 unlock := lock(filepath.Join(goenv.Get("GOCACHE"), fileHash+".c.lock"))
68 defer unlock()
69 70 // Create cache key for the dependencies file.
71 buf, err := json.Marshal(struct {
72 Path string
73 Hash string
74 Flags []string
75 LLVMVersion string
76 }{
77 Path: abspath,
78 Hash: fileHash,
79 Flags: cflags,
80 LLVMVersion: llvm.Version,
81 })
82 if err != nil {
83 panic(err) // shouldn't happen
84 }
85 depfileNameHashBuf := sha512.Sum512_224(buf)
86 depfileNameHash := hex.EncodeToString(depfileNameHashBuf[:])
87 88 // Load dependencies file, if possible.
89 srcLabel := cacheSafeLabel(abspath)
90 depfileName := "dep-" + srcLabel + "-" + depfileNameHash + ".json"
91 depfileCachePath := filepath.Join(goenv.Get("GOCACHE"), depfileName)
92 depfileBuf, err := os.ReadFile(depfileCachePath)
93 var dependencies []string // sorted list of dependency paths
94 if err == nil {
95 // There is a dependency file, that's great!
96 // Parse it first.
97 err := json.Unmarshal(depfileBuf, &dependencies)
98 if err != nil {
99 return "", fmt.Errorf("could not parse dependencies JSON: %w", err)
100 }
101 102 // Obtain hashes of all the files listed as a dependency.
103 outpath, err := makeCFileCachePath(dependencies, depfileNameHash, srcLabel)
104 if err == nil {
105 if _, err := os.Stat(outpath); err == nil {
106 return outpath, nil
107 } else if !errors.Is(err, fs.ErrNotExist) {
108 return "", err
109 }
110 }
111 } else if !errors.Is(err, fs.ErrNotExist) {
112 // expected either nil or IsNotExist
113 return "", err
114 }
115 116 objTmpFile, err := os.CreateTemp(goenv.Get("GOCACHE"), "tmp-*.bc")
117 if err != nil {
118 return "", err
119 }
120 objTmpFile.Close()
121 depTmpFile, err := os.CreateTemp(tmpdir, "dep-*.d")
122 if err != nil {
123 return "", err
124 }
125 depTmpFile.Close()
126 flags := append([]string{}, cflags...) // copy cflags
127 flags = append(flags, "-MD", "-MV", "-MTdeps", "-MF", depTmpFile.Name(), "-flto=thin") // autogenerate dependencies
128 flags = append(flags, "-c", "-o", objTmpFile.Name(), abspath)
129 if strings.ToLower(filepath.Ext(abspath)) == ".s" {
130 // If this is an assembly file (.s or .S, lowercase or uppercase), then
131 // we'll need to add -Qunused-arguments because many parameters are
132 // relevant to C, not assembly. And with -Werror, having meaningless
133 // flags (for the assembler) is a compiler error.
134 flags = append(flags, "-Qunused-arguments")
135 }
136 if printCommands != nil {
137 printCommands("clang", flags...)
138 }
139 err = runCCompiler(flags...)
140 if err != nil {
141 return "", &commandError{"failed to build", abspath, err}
142 }
143 144 // Create sorted and uniqued slice of dependencies.
145 dependencyPaths, err := readDepFile(depTmpFile.Name())
146 if err != nil {
147 return "", err
148 }
149 dependencyPaths = append(dependencyPaths, abspath) // necessary for .s files
150 dependencySet := make(map[string]struct{}, len(dependencyPaths))
151 var dependencySlice []string
152 for _, path := range dependencyPaths {
153 if _, ok := dependencySet[path]; ok {
154 continue
155 }
156 dependencySet[path] = struct{}{}
157 dependencySlice = append(dependencySlice, path)
158 }
159 sort.Strings(dependencySlice)
160 161 // Write dependencies file.
162 f, err := os.CreateTemp(filepath.Dir(depfileCachePath), depfileName)
163 if err != nil {
164 return "", err
165 }
166 167 buf, err = json.MarshalIndent(dependencySlice, "", "\t")
168 if err != nil {
169 panic(err) // shouldn't happen
170 }
171 _, err = f.Write(buf)
172 if err != nil {
173 return "", err
174 }
175 err = f.Close()
176 if err != nil {
177 return "", err
178 }
179 err = os.Rename(f.Name(), depfileCachePath)
180 if err != nil {
181 return "", err
182 }
183 184 // Move temporary object file to final location.
185 outpath, err := makeCFileCachePath(dependencySlice, depfileNameHash, srcLabel)
186 if err != nil {
187 return "", err
188 }
189 err = os.Rename(objTmpFile.Name(), outpath)
190 if err != nil {
191 return "", err
192 }
193 194 return outpath, nil
195 }
196 197 // Create a cache path (a path in GOCACHE) to store the output of a compiler
198 // job. This path is based on the dep file name (which is a hash of metadata
199 // including compiler flags) and the hash of all input files in the paths slice.
200 func makeCFileCachePath(paths []string, depfileNameHash, srcLabel string) (string, error) {
201 // Hash all input files.
202 fileHashes := make(map[string]string, len(paths))
203 for _, path := range paths {
204 hash, err := hashFile(path)
205 if err != nil {
206 return "", err
207 }
208 fileHashes[path] = hash
209 }
210 211 // Calculate a cache key based on the above hashes.
212 buf, err := json.Marshal(struct {
213 DepfileHash string
214 FileHashes map[string]string
215 }{
216 DepfileHash: depfileNameHash,
217 FileHashes: fileHashes,
218 })
219 if err != nil {
220 panic(err) // shouldn't happen
221 }
222 outFileNameBuf := sha512.Sum512_224(buf)
223 cacheKey := hex.EncodeToString(outFileNameBuf[:])
224 225 outpath := filepath.Join(goenv.Get("GOCACHE"), "obj-"+srcLabel+"-"+cacheKey+".bc")
226 return outpath, nil
227 }
228 229 func cacheSafeLabel(abspath string) string {
230 base := filepath.Base(abspath)
231 ext := filepath.Ext(base)
232 name := strings.TrimSuffix(base, ext)
233 var b strings.Builder
234 for _, r := range name {
235 if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' || r == '-' {
236 b.WriteRune(r)
237 }
238 }
239 label := b.String()
240 if len(label) > 40 {
241 label = label[:40]
242 }
243 if label == "" {
244 label = "unknown"
245 }
246 return label
247 }
248 249 // hashFile hashes the given file path and returns the hash as a hex string.
250 func hashFile(path string) (string, error) {
251 f, err := os.Open(path)
252 if err != nil {
253 return "", fmt.Errorf("failed to hash file: %w", err)
254 }
255 defer f.Close()
256 fileHasher := sha512.New512_224()
257 _, err = io.Copy(fileHasher, f)
258 if err != nil {
259 return "", fmt.Errorf("failed to hash file: %w", err)
260 }
261 return hex.EncodeToString(fileHasher.Sum(nil)), nil
262 }
263 264 // readDepFile reads a dependency file in NMake (Visual Studio make) format. The
265 // file is assumed to have a single target named deps.
266 //
267 // There are roughly three make syntax variants:
268 // - BSD make, which doesn't support any escaping. This means that many special
269 // characters are not supported in file names.
270 // - GNU make, which supports escaping using a backslash but when it fails to
271 // find a file it tries to fall back with the literal path name (to match BSD
272 // make).
273 // - NMake (Visual Studio) and Jom, which simply quote the string if there are
274 // any weird characters.
275 //
276 // Clang supports two variants: a format that's a compromise between BSD and GNU
277 // make (and is buggy to match GCC which is equally buggy), and NMake/Jom, which
278 // is at least somewhat sane. This last format isn't perfect either: it does not
279 // correctly handle filenames with quote marks in them. Those are generally not
280 // allowed on Windows, but of course can be used on POSIX like systems. Still,
281 // it's the most sane of any of the formats so readDepFile will use that format.
282 func readDepFile(filename string) ([]string, error) {
283 buf, err := os.ReadFile(filename)
284 if err != nil {
285 return nil, err
286 }
287 if len(buf) == 0 {
288 return nil, nil
289 }
290 return parseDepFile(string(buf))
291 }
292 293 func parseDepFile(s string) ([]string, error) {
294 // This function makes no attempt at parsing anything other than Clang -MD
295 // -MV output.
296 297 // For Windows: replace CRLF with LF to make the logic below simpler.
298 s = strings.ReplaceAll(s, "\r\n", "\n")
299 300 // Collapse all lines ending in a backslash. These backslashes are really
301 // just a way to continue a line without making very long lines.
302 s = strings.ReplaceAll(s, "\\\n", " ")
303 304 // Only use the first line, which is expected to begin with "deps:".
305 line := strings.SplitN(s, "\n", 2)[0]
306 if !strings.HasPrefix(line, "deps:") {
307 return nil, errors.New("readDepFile: expected 'deps:' prefix")
308 }
309 line = strings.TrimSpace(line[len("deps:"):])
310 311 var deps []string
312 for line != "" {
313 if line[0] == '"' {
314 // File path is quoted. Path ends with double quote.
315 // This does not handle double quotes in path names, which is a
316 // problem on non-Windows systems.
317 line = line[1:]
318 end := strings.IndexByte(line, '"')
319 if end < 0 {
320 return nil, errors.New("readDepFile: path is incorrectly quoted")
321 }
322 dep := line[:end]
323 line = strings.TrimSpace(line[end+1:])
324 deps = append(deps, dep)
325 } else {
326 // File path is not quoted. Path ends in space or EOL.
327 end := strings.IndexFunc(line, unicode.IsSpace)
328 if end < 0 {
329 // last dependency
330 deps = append(deps, line)
331 break
332 }
333 dep := line[:end]
334 line = strings.TrimSpace(line[end:])
335 deps = append(deps, dep)
336 }
337 }
338 return deps, nil
339 }
340