1 #!/usr/bin/env bash
2 export LC_ALL=C
3 set -e -o pipefail
4 5 # Source the common prelude, which:
6 # 1. Checks if we're at the top directory of the Bitcoin Core repository
7 # 2. Defines a few common functions and variables
8 #
9 # shellcheck source=libexec/prelude.bash
10 source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash"
11 12 13 ###################
14 ## SANITY CHECKS ##
15 ###################
16 17 ################
18 # Required non-builtin commands should be invocable
19 ################
20 21 check_tools cat mkdir make getent curl git guix
22 23 ################
24 # GUIX_BUILD_OPTIONS should be empty
25 ################
26 #
27 # GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that
28 # can perform builds. This seems like what we want instead of
29 # ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually
30 # _appended_ to normal command-line options. Meaning that they will take
31 # precedence over the command-specific ADDITIONAL_GUIX_<CMD>_FLAGS.
32 #
33 # This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's
34 # existence here and direct users of this script to use our (more flexible)
35 # custom environment variables.
36 if [ -n "$GUIX_BUILD_OPTIONS" ]; then
37 cat << EOF
38 Error: Environment variable GUIX_BUILD_OPTIONS is not empty:
39 '$GUIX_BUILD_OPTIONS'
40 41 Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset
42 GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options
43 across guix commands or ADDITIONAL_GUIX_<CMD>_FLAGS to set build options for a
44 specific guix command.
45 46 See contrib/guix/README.md for more details.
47 EOF
48 exit 1
49 fi
50 51 ################
52 # The git worktree should not be dirty
53 ################
54 55 if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then
56 cat << EOF
57 ERR: The current git worktree is dirty, which may lead to broken builds.
58 59 Aborting...
60 61 Hint: To make your git worktree clean, You may want to:
62 1. Commit your changes,
63 2. Stash your changes, or
64 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on
65 using a dirty worktree
66 EOF
67 exit 1
68 fi
69 70 mkdir -p "$VERSION_BASE"
71 72 ################
73 # SOURCE_DATE_EPOCH should not unintentionally be set
74 ################
75 76 check_source_date_epoch
77 78 ################
79 # Build directories should not exist
80 ################
81 82 # Default to building for all supported HOSTs (overridable by environment)
83 # powerpc64le-linux-gnu currently disabled due non-determinism issues across build arches.
84 export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu
85 x86_64-w64-mingw32
86 x86_64-apple-darwin arm64-apple-darwin}"
87 88 # Accumulate a list of build directories that already exist...
89 hosts_distsrc_exists=""
90 for host in $HOSTS; do
91 if [ -e "$(distsrc_for_host "$host")" ]; then
92 hosts_distsrc_exists+=" ${host}"
93 fi
94 done
95 96 if [ -n "$hosts_distsrc_exists" ]; then
97 # ...so that we can print them out nicely in an error message
98 cat << EOF
99 ERR: Build directories for this commit already exist for the following platform
100 triples you're attempting to build, probably because of previous builds.
101 Please remove, or otherwise deal with them prior to starting another build.
102 103 Aborting...
104 105 Hint: To blow everything away, you may want to use:
106 107 $ ./contrib/guix/guix-clean
108 109 Specifically, this will remove all files without an entry in the index,
110 excluding the SDK directory, the depends download cache, the depends built
111 packages cache, the garbage collector roots for Guix environments, and the
112 output directory.
113 EOF
114 for host in $hosts_distsrc_exists; do
115 echo " ${host} '$(distsrc_for_host "$host")'"
116 done
117 exit 1
118 else
119 mkdir -p "$DISTSRC_BASE"
120 fi
121 122 ################
123 # When building for darwin, the macOS SDK should exist
124 ################
125 126 for host in $HOSTS; do
127 case "$host" in
128 *darwin*)
129 OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=@@g')"
130 if [ -e "$OSX_SDK" ]; then
131 echo "Found macOS SDK at '${OSX_SDK}', using..."
132 break
133 else
134 echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, or define SDK_PATH environment variable. Exiting..."
135 exit 1
136 fi
137 ;;
138 esac
139 done
140 141 ################
142 # VERSION_BASE should have enough space
143 ################
144 145 avail_KiB="$(df -Pk "$VERSION_BASE" | sed 1d | tr -s ' ' | cut -d' ' -f4)"
146 total_required_KiB=0
147 for host in $HOSTS; do
148 case "$host" in
149 *darwin*) required_KiB=440000 ;;
150 *mingw*) required_KiB=7600000 ;;
151 *) required_KiB=6400000 ;;
152 esac
153 total_required_KiB=$((total_required_KiB+required_KiB))
154 done
155 156 if (( total_required_KiB > avail_KiB )); then
157 total_required_GiB=$((total_required_KiB / 1048576))
158 avail_GiB=$((avail_KiB / 1048576))
159 echo "Performing a Bitcoin Core Guix build for the selected HOSTS requires ${total_required_GiB} GiB, however, only ${avail_GiB} GiB is available. Please free up some disk space before performing the build."
160 exit 1
161 fi
162 163 ################
164 # Check that we can connect to the guix-daemon
165 ################
166 167 cat << EOF
168 Checking that we can connect to the guix-daemon...
169 170 Hint: If this hangs, you may want to try turning your guix-daemon off and on
171 again.
172 173 EOF
174 if ! guix gc --list-failures > /dev/null; then
175 cat << EOF
176 177 ERR: Failed to connect to the guix-daemon, please ensure that one is running and
178 reachable.
179 EOF
180 exit 1
181 fi
182 183 # Developer note: we could use `guix repl` for this check and run:
184 #
185 # (import (guix store)) (close-connection (open-connection))
186 #
187 # However, the internal API is likely to change more than the CLI invocation
188 189 ################
190 # Services database must have basic entries
191 ################
192 193 if ! getent services http https ftp > /dev/null 2>&1; then
194 cat << EOF
195 ERR: Your system's C library cannot find service database entries for at least
196 one of the following services: http, https, ftp.
197 198 Hint: Most likely, /etc/services does not exist yet (common for docker images
199 and minimal distros), or you don't have permissions to access it.
200 201 If /etc/services does not exist yet, you may want to install the
202 appropriate package for your distro which provides it.
203 204 On Debian/Ubuntu: netbase
205 On Arch Linux: iana-etc
206 207 For more information, see: getent(1), services(5)
208 209 EOF
210 211 fi
212 213 #########
214 # SETUP #
215 #########
216 217 # Determine the maximum number of jobs to run simultaneously (overridable by
218 # environment)
219 JOBS="${JOBS:-$(nproc)}"
220 221 # Usage: host_to_commonname HOST
222 #
223 # HOST: The current platform triple we're building for
224 #
225 host_to_commonname() {
226 case "$1" in
227 *darwin*) echo osx ;;
228 *mingw*) echo win ;;
229 *linux*) echo linux ;;
230 *) exit 1 ;;
231 esac
232 }
233 234 # Determine the reference time used for determinism (overridable by environment)
235 SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}"
236 237 # Precious directories are those which should not be cleaned between successive
238 # guix builds
239 depends_precious_dir_names="SOURCES_PATH BASE_CACHE SDK_PATH"
240 base_dir_names="OUTDIR_BASE PROFILES_BASE"
241 precious_dir_names="${depends_precious_dir_names} ${base_dir_names}"
242 243 # If the user explicitly specified a precious directory, create it so we
244 # can map it into the container
245 for precious_dir_name in $precious_dir_names; do
246 precious_dir_path="${!precious_dir_name}"
247 if [ -n "$precious_dir_path" ]; then
248 if [ ! -e "$precious_dir_path" ]; then
249 mkdir -p "$precious_dir_path"
250 elif [ -L "$precious_dir_path" ]; then
251 echo "ERR: ${precious_dir_name} cannot be a symbolic link"
252 exit 1
253 elif [ ! -d "$precious_dir_path" ]; then
254 echo "ERR: ${precious_dir_name} must be a directory"
255 exit 1
256 fi
257 fi
258 done
259 260 mkdir -p "$VAR_BASE"
261 262 # Record the _effective_ values of precious directories such that guix-clean can
263 # avoid clobbering them if appropriate.
264 #
265 # shellcheck disable=SC2046,SC2086
266 {
267 # Get depends precious dir definitions from depends
268 make -C "${PWD}/depends" \
269 --no-print-directory \
270 -- $(printf "print-%s\n" $depends_precious_dir_names)
271 272 # Get remaining precious dir definitions from the environment
273 for precious_dir_name in $base_dir_names; do
274 precious_dir_path="${!precious_dir_name}"
275 echo "${precious_dir_name}=${precious_dir_path}"
276 done
277 } > "${VAR_BASE}/precious_dirs"
278 279 # Make sure an output directory exists for our builds
280 OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}"
281 mkdir -p "$OUTDIR_BASE"
282 283 # Download the depends sources now as we won't have internet access in the build
284 # container
285 for host in $HOSTS; do
286 make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"}
287 done
288 289 290 #########
291 # BUILD #
292 #########
293 294 # Function to be called when building for host ${1} and the user interrupts the
295 # build
296 int_trap() {
297 cat << EOF
298 ** INT received while building ${1}, you may want to clean up the relevant
299 work directories (e.g. distsrc-*) before rebuilding
300 301 Hint: To blow everything away, you may want to use:
302 303 $ ./contrib/guix/guix-clean
304 305 Specifically, this will remove all files without an entry in the index,
306 excluding the SDK directory, the depends download cache, the depends built
307 packages cache, the garbage collector roots for Guix environments, and the
308 output directory.
309 EOF
310 }
311 312 # Deterministically build Bitcoin Core
313 # shellcheck disable=SC2153
314 for host in $HOSTS; do
315 316 # Display proper warning when the user interrupts the build
317 trap 'int_trap ${host}' INT
318 319 (
320 # Required for 'contrib/guix/manifest.scm' to output the right manifest
321 # for the particular $HOST we're building for
322 export HOST="$host"
323 324 cat << EOF
325 INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}:
326 ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set}
327 ...running at most ${JOBS:?not set} jobs
328 ...from worktree directory: '${PWD}'
329 ...bind-mounted in container to: '/bitcoin'
330 ...in build directory: '$(distsrc_for_host "$HOST")'
331 ...bind-mounted in container to: '$(distsrc_for_host "$HOST" "" /distsrc-base)'
332 ...outputting in: '$(outdir_for_host "$HOST")'
333 ...bind-mounted in container to: '$(outdir_for_host "$HOST" "" /outdir-base)'
334 ADDITIONAL FLAGS (if set)
335 ADDITIONAL_GUIX_COMMON_FLAGS: ${ADDITIONAL_GUIX_COMMON_FLAGS}
336 ADDITIONAL_GUIX_ENVIRONMENT_FLAGS: ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS}
337 ADDITIONAL_GUIX_TIMEMACHINE_FLAGS: ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS}
338 EOF
339 340 # Run the build script 'contrib/guix/libexec/build.sh' in the build
341 # container specified by 'contrib/guix/manifest.scm'.
342 #
343 # Explanation of `guix shell` flags:
344 #
345 # --container run command within an isolated container
346 #
347 # Running in an isolated container minimizes build-time differences
348 # between machines and improves reproducibility
349 #
350 # --writable-root make the root filesystem writable
351 #
352 # --pure unset existing environment variables
353 #
354 # Same rationale as --container
355 #
356 # --no-cwd do not share current working directory with an
357 # isolated container
358 #
359 # When --container is specified, the default behavior is to share
360 # the current working directory with the isolated container at the
361 # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to
362 # '/home/satoshi/bitcoin/'). This means that the $PWD inside the
363 # container becomes a source of irreproducibility. --no-cwd disables
364 # this behaviour.
365 #
366 # --share=SPEC for containers, share writable host file system
367 # according to SPEC
368 #
369 # --share="$PWD"=/bitcoin
370 #
371 # maps our current working directory to /bitcoin
372 # inside the isolated container, which we later cd
373 # into.
374 #
375 # While we don't want to map our current working directory to the
376 # same exact path (as this introduces irreproducibility), we do want
377 # it to be at a _fixed_ path _somewhere_ inside the isolated
378 # container so that we have something to build. '/bitcoin' was
379 # chosen arbitrarily.
380 #
381 # ${SOURCES_PATH:+--share="$SOURCES_PATH"}
382 #
383 # make the downloaded depends sources path available
384 # inside the isolated container
385 #
386 # The isolated container has no network access as it's in a
387 # different network namespace from the main machine, so we have to
388 # make the downloaded depends sources available to it. The sources
389 # should have been downloaded prior to this invocation.
390 #
391 # --keep-failed keep build tree of failed builds
392 #
393 # When builds of the Guix environment itself (not Bitcoin Core)
394 # fail, it is useful for the build tree to be kept for debugging
395 # purposes.
396 #
397 # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"}
398 #
399 # fetch substitute from SUBSTITUTE_URLS if they are
400 # authorized
401 #
402 # Depending on the user's security model, it may be desirable to use
403 # substitutes (pre-built packages) from servers that the user trusts.
404 # Please read the README.md in the same directory as this file for
405 # more information.
406 #
407 # shellcheck disable=SC2086
408 time-machine shell --manifest="${PWD}/contrib/guix/manifest_build.scm" \
409 --container \
410 --writable-root \
411 --pure \
412 --no-cwd \
413 --share="$PWD"=/bitcoin \
414 --share="$DISTSRC_BASE"=/distsrc-base \
415 --share="$OUTDIR_BASE"=/outdir-base \
416 --expose="$(git rev-parse --git-common-dir)" \
417 ${SOURCES_PATH:+--share="$SOURCES_PATH"} \
418 ${BASE_CACHE:+--share="$BASE_CACHE"} \
419 ${SDK_PATH:+--share="$SDK_PATH"} \
420 --cores="$JOBS" \
421 --keep-failed \
422 --fallback \
423 --link-profile \
424 --root="$(profiledir_for_host "${HOST}")" \
425 ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \
426 ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \
427 -- env HOST="$host" \
428 DISTNAME="$DISTNAME" \
429 JOBS="$JOBS" \
430 SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \
431 ${V:+V=1} \
432 ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \
433 ${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \
434 ${SDK_PATH:+SDK_PATH="$SDK_PATH"} \
435 DISTSRC="$(distsrc_for_host "$HOST" "" /distsrc-base)" \
436 OUTDIR="$(outdir_for_host "$HOST" "" /outdir-base)" \
437 DIST_ARCHIVE_BASE=/outdir-base/dist-archive \
438 bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh"
439 )
440 441 done
442