1 #!/usr/bin/env bash
2 export LC_ALL=C
3 set -e -o pipefail
4 5 # Source the common prelude, which:
6 # 1. Checks if we're at the top directory of the Limenka repository
7 # 2. Defines a few common functions and variables
8 #
9 # shellcheck source=libexec/prelude.bash
10 source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash"
11 12 13 ###################
14 ## SANITY CHECKS ##
15 ###################
16 17 ################
18 # Required non-builtin commands should be invocable
19 ################
20 21 check_tools cat mkdir make getent curl git guix
22 23 ################
24 # GUIX_BUILD_OPTIONS should be empty
25 ################
26 #
27 # GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that
28 # can perform builds. This seems like what we want instead of
29 # ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually
30 # _appended_ to normal command-line options. Meaning that they will take
31 # precedence over the command-specific ADDITIONAL_GUIX_<CMD>_FLAGS.
32 #
33 # This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's
34 # existence here and direct users of this script to use our (more flexible)
35 # custom environment variables.
36 if [ -n "$GUIX_BUILD_OPTIONS" ]; then
37 cat << EOF
38 Error: Environment variable GUIX_BUILD_OPTIONS is not empty:
39 '$GUIX_BUILD_OPTIONS'
40 41 Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset
42 GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options
43 across guix commands or ADDITIONAL_GUIX_<CMD>_FLAGS to set build options for a
44 specific guix command.
45 46 See contrib/guix/README.md for more details.
47 EOF
48 exit 1
49 fi
50 51 ################
52 # The git worktree should not be dirty
53 ################
54 55 if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then
56 cat << EOF
57 ERR: The current git worktree is dirty, which may lead to broken builds.
58 59 Aborting...
60 61 Hint: To make your git worktree clean, You may want to:
62 1. Commit your changes,
63 2. Stash your changes, or
64 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on
65 using a dirty worktree
66 EOF
67 exit 1
68 fi
69 70 mkdir -p "$VERSION_BASE"
71 72 ################
73 # SOURCE_DATE_EPOCH should not unintentionally be set
74 ################
75 76 check_source_date_epoch
77 78 ################
79 # Build directories should not exist
80 ################
81 82 # Default to building for all supported HOSTs (overridable by environment)
83 export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu
84 x86_64-w64-mingw32
85 x86_64-apple-darwin arm64-apple-darwin}"
86 87 # Usage: distsrc_for_host HOST
88 #
89 # HOST: The current platform triple we're building for
90 #
91 distsrc_for_host() {
92 echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}"
93 }
94 95 # Accumulate a list of build directories that already exist...
96 hosts_distsrc_exists=""
97 for host in $HOSTS; do
98 if [ -e "$(distsrc_for_host "$host")" ]; then
99 hosts_distsrc_exists+=" ${host}"
100 fi
101 done
102 103 if [ -n "$hosts_distsrc_exists" ]; then
104 # ...so that we can print them out nicely in an error message
105 cat << EOF
106 ERR: Build directories for this commit already exist for the following platform
107 triples you're attempting to build, probably because of previous builds.
108 Please remove, or otherwise deal with them prior to starting another build.
109 110 Aborting...
111 112 Hint: To blow everything away, you may want to use:
113 114 $ ./contrib/guix/guix-clean
115 116 Specifically, this will remove all files without an entry in the index,
117 excluding the SDK directory, the depends download cache, the depends built
118 packages cache, the garbage collector roots for Guix environments, and the
119 output directory.
120 EOF
121 for host in $hosts_distsrc_exists; do
122 echo " ${host} '$(distsrc_for_host "$host")'"
123 done
124 exit 1
125 else
126 mkdir -p "$DISTSRC_BASE"
127 fi
128 129 ################
130 # When building for darwin, the macOS SDK should exist
131 ################
132 133 for host in $HOSTS; do
134 case "$host" in
135 *darwin*)
136 OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=@@g')"
137 if [ -e "$OSX_SDK" ]; then
138 echo "Found macOS SDK at '${OSX_SDK}', using..."
139 break
140 else
141 echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, or define SDK_PATH environment variable. Exiting..."
142 exit 1
143 fi
144 ;;
145 esac
146 done
147 148 ################
149 # VERSION_BASE should have enough space
150 ################
151 152 avail_KiB="$(df -Pk "$VERSION_BASE" | sed 1d | tr -s ' ' | cut -d' ' -f4)"
153 total_required_KiB=0
154 for host in $HOSTS; do
155 case "$host" in
156 *darwin*) required_KiB=440000 ;;
157 *mingw*) required_KiB=7600000 ;;
158 *) required_KiB=6400000 ;;
159 esac
160 total_required_KiB=$((total_required_KiB+required_KiB))
161 done
162 163 if (( total_required_KiB > avail_KiB )); then
164 total_required_GiB=$((total_required_KiB / 1048576))
165 avail_GiB=$((avail_KiB / 1048576))
166 echo "Performing a Limenka Guix build for the selected HOSTS requires ${total_required_GiB} GiB, however, only ${avail_GiB} GiB is available. Please free up some disk space before performing the build."
167 exit 1
168 fi
169 170 ################
171 # Check that we can connect to the guix-daemon
172 ################
173 174 cat << EOF
175 Checking that we can connect to the guix-daemon...
176 177 Hint: If this hangs, you may want to try turning your guix-daemon off and on
178 again.
179 180 EOF
181 if ! guix gc --list-failures > /dev/null; then
182 cat << EOF
183 184 ERR: Failed to connect to the guix-daemon, please ensure that one is running and
185 reachable.
186 EOF
187 exit 1
188 fi
189 190 # Developer note: we could use `guix repl` for this check and run:
191 #
192 # (import (guix store)) (close-connection (open-connection))
193 #
194 # However, the internal API is likely to change more than the CLI invocation
195 196 ################
197 # Services database must have basic entries
198 ################
199 200 if ! getent services http https ftp > /dev/null 2>&1; then
201 cat << EOF
202 ERR: Your system's C library cannot find service database entries for at least
203 one of the following services: http, https, ftp.
204 205 Hint: Most likely, /etc/services does not exist yet (common for docker images
206 and minimal distros), or you don't have permissions to access it.
207 208 If /etc/services does not exist yet, you may want to install the
209 appropriate package for your distro which provides it.
210 211 On Debian/Ubuntu: netbase
212 On Arch Linux: iana-etc
213 214 For more information, see: getent(1), services(5)
215 216 EOF
217 218 fi
219 220 #########
221 # SETUP #
222 #########
223 224 # Determine the maximum number of jobs to run simultaneously (overridable by
225 # environment)
226 JOBS="${JOBS:-$(nproc)}"
227 228 # Usage: host_to_commonname HOST
229 #
230 # HOST: The current platform triple we're building for
231 #
232 host_to_commonname() {
233 case "$1" in
234 *darwin*) echo osx ;;
235 *mingw*) echo win ;;
236 *linux*) echo linux ;;
237 *) exit 1 ;;
238 esac
239 }
240 241 # Determine the reference time used for determinism (overridable by environment)
242 SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}"
243 244 # Precious directories are those which should not be cleaned between successive
245 # guix builds
246 depends_precious_dir_names='SOURCES_PATH BASE_CACHE SDK_PATH'
247 precious_dir_names="${depends_precious_dir_names} OUTDIR_BASE PROFILES_BASE"
248 249 # Usage: contains IFS-SEPARATED-LIST ITEM
250 contains() {
251 for i in ${1}; do
252 if [ "$i" = "${2}" ]; then
253 return 0 # Found!
254 fi
255 done
256 return 1
257 }
258 259 # If the user explicitly specified a precious directory, create it so we
260 # can map it into the container
261 for precious_dir_name in $precious_dir_names; do
262 precious_dir_path="${!precious_dir_name}"
263 if [ -n "$precious_dir_path" ]; then
264 if [ ! -e "$precious_dir_path" ]; then
265 mkdir -p "$precious_dir_path"
266 elif [ -L "$precious_dir_path" ]; then
267 echo "ERR: ${precious_dir_name} cannot be a symbolic link"
268 exit 1
269 elif [ ! -d "$precious_dir_path" ]; then
270 echo "ERR: ${precious_dir_name} must be a directory"
271 exit 1
272 fi
273 fi
274 done
275 276 mkdir -p "$VAR_BASE"
277 278 # Record the _effective_ values of precious directories such that guix-clean can
279 # avoid clobbering them if appropriate.
280 #
281 # shellcheck disable=SC2046,SC2086
282 {
283 # Get depends precious dir definitions from depends
284 make -C "${PWD}/depends" \
285 --no-print-directory \
286 -- $(printf "print-%s\n" $depends_precious_dir_names)
287 288 # Get remaining precious dir definitions from the environment
289 for precious_dir_name in $precious_dir_names; do
290 precious_dir_path="${!precious_dir_name}"
291 if ! contains "$depends_precious_dir_names" "$precious_dir_name"; then
292 echo "${precious_dir_name}=${precious_dir_path}"
293 fi
294 done
295 } > "${VAR_BASE}/precious_dirs"
296 297 # Make sure an output directory exists for our builds
298 OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}"
299 mkdir -p "$OUTDIR_BASE"
300 301 # Download the depends sources now as we won't have internet access in the build
302 # container
303 for host in $HOSTS; do
304 make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"}
305 done
306 307 # Usage: outdir_for_host HOST SUFFIX
308 #
309 # HOST: The current platform triple we're building for
310 #
311 outdir_for_host() {
312 echo "${OUTDIR_BASE}/${1}${2:+-${2}}"
313 }
314 315 # Usage: profiledir_for_host HOST SUFFIX
316 #
317 # HOST: The current platform triple we're building for
318 #
319 profiledir_for_host() {
320 echo "${PROFILES_BASE}/${1}${2:+-${2}}"
321 }
322 323 324 #########
325 # BUILD #
326 #########
327 328 # Function to be called when building for host ${1} and the user interrupts the
329 # build
330 int_trap() {
331 cat << EOF
332 ** INT received while building ${1}, you may want to clean up the relevant
333 work directories (e.g. distsrc-*) before rebuilding
334 335 Hint: To blow everything away, you may want to use:
336 337 $ ./contrib/guix/guix-clean
338 339 Specifically, this will remove all files without an entry in the index,
340 excluding the SDK directory, the depends download cache, the depends built
341 packages cache, the garbage collector roots for Guix environments, and the
342 output directory.
343 EOF
344 }
345 346 guix-prefetch '1h00qp4z5k6lfz310xjwsmqs8fwxi6ngas51169cafz4h9fmc68y' 'https://archives.boost.io/release/1.80.0/source/boost_1_80_0.tar.bz2'
347 348 # Deterministically build Limenka
349 # shellcheck disable=SC2153
350 for host in $HOSTS; do
351 352 # Display proper warning when the user interrupts the build
353 trap 'int_trap ${host}' INT
354 355 (
356 # Required for 'contrib/guix/manifest.scm' to output the right manifest
357 # for the particular $HOST we're building for
358 export HOST="$host"
359 360 # shellcheck disable=SC2030
361 cat << EOF
362 INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}:
363 ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set}
364 ...running at most ${JOBS:?not set} jobs
365 ...from worktree directory: '${PWD}'
366 ...bind-mounted in container to: '/limenka'
367 ...in build directory: '$(distsrc_for_host "$HOST")'
368 ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")'
369 ...outputting in: '$(outdir_for_host "$HOST")'
370 ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")'
371 ADDITIONAL FLAGS (if set)
372 ADDITIONAL_GUIX_COMMON_FLAGS: ${ADDITIONAL_GUIX_COMMON_FLAGS}
373 ADDITIONAL_GUIX_ENVIRONMENT_FLAGS: ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS}
374 ADDITIONAL_GUIX_TIMEMACHINE_FLAGS: ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS}
375 EOF
376 377 # Run the build script 'contrib/guix/libexec/build.sh' in the build
378 # container specified by 'contrib/guix/manifest.scm'.
379 #
380 # Explanation of `guix shell` flags:
381 #
382 # --container run command within an isolated container
383 #
384 # Running in an isolated container minimizes build-time differences
385 # between machines and improves reproducibility
386 #
387 # --pure unset existing environment variables
388 #
389 # Same rationale as --container
390 #
391 # --no-cwd do not share current working directory with an
392 # isolated container
393 #
394 # When --container is specified, the default behavior is to share
395 # the current working directory with the isolated container at the
396 # same exact path (e.g. mapping '/home/satoshi/limenka/' to
397 # '/home/satoshi/limenka/'). This means that the $PWD inside the
398 # container becomes a source of irreproducibility. --no-cwd disables
399 # this behaviour.
400 #
401 # --share=SPEC for containers, share writable host file system
402 # according to SPEC
403 #
404 # --share="$PWD"=/limenka
405 #
406 # maps our current working directory to /limenka
407 # inside the isolated container, which we later cd
408 # into.
409 #
410 # While we don't want to map our current working directory to the
411 # same exact path (as this introduces irreproducibility), we do want
412 # it to be at a _fixed_ path _somewhere_ inside the isolated
413 # container so that we have something to build. '/limenka' was
414 # chosen arbitrarily.
415 #
416 # ${SOURCES_PATH:+--share="$SOURCES_PATH"}
417 #
418 # make the downloaded depends sources path available
419 # inside the isolated container
420 #
421 # The isolated container has no network access as it's in a
422 # different network namespace from the main machine, so we have to
423 # make the downloaded depends sources available to it. The sources
424 # should have been downloaded prior to this invocation.
425 #
426 # --keep-failed keep build tree of failed builds
427 #
428 # When builds of the Guix environment itself (not Limenka)
429 # fail, it is useful for the build tree to be kept for debugging
430 # purposes.
431 #
432 # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"}
433 #
434 # fetch substitute from SUBSTITUTE_URLS if they are
435 # authorized
436 #
437 # Depending on the user's security model, it may be desirable to use
438 # substitutes (pre-built packages) from servers that the user trusts.
439 # Please read the README.md in the same directory as this file for
440 # more information.
441 #
442 # shellcheck disable=SC2086,SC2031
443 time-machine shell --manifest="${PWD}/contrib/guix/manifest.scm" \
444 --container \
445 --pure \
446 --no-cwd \
447 --share="$PWD"=/limenka \
448 --share="$DISTSRC_BASE"=/distsrc-base \
449 --share="$OUTDIR_BASE"=/outdir-base \
450 --expose="$(git rev-parse --git-common-dir)" \
451 ${SOURCES_PATH:+--share="$SOURCES_PATH"} \
452 ${BASE_CACHE:+--share="$BASE_CACHE"} \
453 ${SDK_PATH:+--share="$SDK_PATH"} \
454 --cores="$JOBS" \
455 --keep-failed \
456 --fallback \
457 --link-profile \
458 --root="$(profiledir_for_host "${HOST}")" \
459 ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \
460 ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \
461 -- env HOST="$host" \
462 DISTNAME="$DISTNAME" \
463 JOBS="$JOBS" \
464 SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \
465 ${V:+V=1} \
466 ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \
467 ${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \
468 ${SDK_PATH:+SDK_PATH="$SDK_PATH"} \
469 DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \
470 OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")" \
471 DIST_ARCHIVE_BASE=/outdir-base/dist-archive \
472 bash -c "cd /limenka && bash contrib/guix/libexec/build.sh"
473 )
474 475 done
476