1 #!/usr/bin/env bash
2 #
3 # Copyright (c) 2018-present The Limenka developers
4 # Distributed under the MIT software license, see the accompanying
5 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
6 7 export LC_ALL=C.UTF-8
8 export CI_IMAGE_LABEL="limenka-ci-test"
9 10 set -o errexit -o pipefail -o xtrace
11 12 if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
13 # Export all env vars to avoid missing some.
14 # Though, exclude those with newlines to avoid parsing problems.
15 python3 -c 'import os; [print(f"{key}={value}") for key, value in os.environ.items() if "\n" not in value and "HOME" != key and "PATH" != key and "USER" != key]' | tee "/tmp/env-$USER-$CONTAINER_NAME"
16 17 # Env vars during the build can not be changed. For example, a modified
18 # $MAKEJOBS is ignored in the build process. Use --cpuset-cpus as an
19 # approximation to respect $MAKEJOBS somewhat, if cpuset is available.
20 MAYBE_CPUSET=""
21 if [ "$HAVE_CGROUP_CPUSET" ]; then
22 MAYBE_CPUSET="--cpuset-cpus=$( python3 -c "import random;P=$( nproc );M=min(P,int('$MAKEJOBS'.lstrip('-j')));print(','.join(map(str,sorted(random.sample(range(P),M)))))" )"
23 fi
24 echo "Creating $CI_IMAGE_NAME_TAG container to run in"
25 26 # Use buildx unconditionally
27 # Using buildx is required to properly load the correct driver, for use with registry caching. Neither build, nor BUILDKIT=1 currently do this properly
28 # shellcheck disable=SC2086
29 docker buildx build \
30 --file "${BASE_READ_ONLY_DIR}/ci/test_imagefile" \
31 --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \
32 --build-arg "FILE_ENV=${FILE_ENV}" \
33 --build-arg "BASE_ROOT_DIR=${BASE_ROOT_DIR}" \
34 $MAYBE_CPUSET \
35 --platform="${CI_IMAGE_PLATFORM}" \
36 --label="${CI_IMAGE_LABEL}" \
37 --tag="${CONTAINER_NAME}" \
38 $DOCKER_BUILD_CACHE_ARG \
39 "${BASE_READ_ONLY_DIR}"
40 41 docker volume create "${CONTAINER_NAME}_ccache" || true
42 docker volume create "${CONTAINER_NAME}_depends" || true
43 docker volume create "${CONTAINER_NAME}_depends_sources" || true
44 docker volume create "${CONTAINER_NAME}_previous_releases" || true
45 46 CI_CCACHE_MOUNT="type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR"
47 CI_DEPENDS_MOUNT="type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR/built"
48 CI_DEPENDS_SOURCES_MOUNT="type=volume,src=${CONTAINER_NAME}_depends_sources,dst=$DEPENDS_DIR/sources"
49 CI_PREVIOUS_RELEASES_MOUNT="type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR"
50 CI_BUILD_MOUNT=""
51 52 if [ "$DANGER_CI_ON_HOST_FOLDERS" ]; then
53 # ensure the directories exist
54 mkdir -p "${CCACHE_DIR}"
55 mkdir -p "${DEPENDS_DIR}/built"
56 mkdir -p "${DEPENDS_DIR}/sources"
57 mkdir -p "${PREVIOUS_RELEASES_DIR}"
58 mkdir -p "${BASE_BUILD_DIR}" # Unset by default, must be defined externally
59 60 CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=$CCACHE_DIR"
61 CI_DEPENDS_MOUNT="type=bind,src=${DEPENDS_DIR}/built,dst=$DEPENDS_DIR/built"
62 CI_DEPENDS_SOURCES_MOUNT="type=bind,src=${DEPENDS_DIR}/sources,dst=$DEPENDS_DIR/sources"
63 CI_PREVIOUS_RELEASES_MOUNT="type=bind,src=${PREVIOUS_RELEASES_DIR},dst=$PREVIOUS_RELEASES_DIR"
64 CI_BUILD_MOUNT="--mount type=bind,src=${BASE_BUILD_DIR},dst=${BASE_BUILD_DIR}"
65 fi
66 67 if [ "$DANGER_CI_ON_HOST_CCACHE_FOLDER" ]; then
68 # Temporary exclusion for https://github.com/limenka/limenka/issues/31108
69 # to allow CI configs and envs generated in the past to work for a bit longer.
70 # Can be removed in March 2025.
71 if [ "${CCACHE_DIR}" != "/tmp/ccache_dir" ]; then
72 if [ ! -d "${CCACHE_DIR}" ]; then
73 echo "Error: Directory '${CCACHE_DIR}' must be created in advance."
74 exit 1
75 fi
76 CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=${CCACHE_DIR}"
77 fi # End temporary exclusion
78 fi
79 80 docker network create --ipv6 --subnet 1111:1111::/112 ci-ip6net || true
81 82 if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then
83 echo "Restart docker before run to stop and clear all containers started with --rm"
84 podman container rm --force --all # Similar to "systemctl restart docker"
85 86 # Still prune everything in case the filtered pruning doesn't work, or if labels were not set
87 # on a previous run. Belt and suspenders approach, should be fine to remove in the future.
88 # Prune images used by --external containers (e.g. build containers) when
89 # using podman.
90 echo "Prune all dangling images"
91 podman image prune --force --external
92 fi
93 echo "Prune all dangling $CI_IMAGE_LABEL images"
94 # When detecting podman-docker, `--external` should be added.
95 docker image prune --force --filter "label=$CI_IMAGE_LABEL"
96 97 # Append $USER to /tmp/env to support multi-user systems and $CONTAINER_NAME
98 # to allow support starting multiple runs simultaneously by the same user.
99 # shellcheck disable=SC2086
100 CI_CONTAINER_ID=$(docker run --cap-add LINUX_IMMUTABLE $CI_CONTAINER_CAP --rm --interactive --detach --tty \
101 --mount "type=bind,src=$BASE_READ_ONLY_DIR,dst=$BASE_READ_ONLY_DIR,readonly" \
102 --mount "${CI_CCACHE_MOUNT}" \
103 --mount "${CI_DEPENDS_MOUNT}" \
104 --mount "${CI_DEPENDS_SOURCES_MOUNT}" \
105 --mount "${CI_PREVIOUS_RELEASES_MOUNT}" \
106 ${CI_BUILD_MOUNT} \
107 --env-file /tmp/env-$USER-$CONTAINER_NAME \
108 --name "$CONTAINER_NAME" \
109 --network ci-ip6net \
110 --platform="${CI_IMAGE_PLATFORM}" \
111 "$CONTAINER_NAME")
112 export CI_CONTAINER_ID
113 export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}"
114 else
115 echo "Running on host system without docker wrapper"
116 echo "Create missing folders"
117 mkdir -p "${CCACHE_DIR}"
118 mkdir -p "${PREVIOUS_RELEASES_DIR}"
119 fi
120 121 if [ "$CI_OS_NAME" == "macos" ]; then
122 IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt"
123 export IN_GETOPT_BIN
124 fi
125 126 CI_EXEC () {
127 $CI_EXEC_CMD_PREFIX bash -c "export PATH=\"/path_with space:${BINS_SCRATCH_DIR}:${BASE_ROOT_DIR}/ci/retry:\$PATH\" && cd \"${BASE_ROOT_DIR}\" && $*"
128 }
129 export -f CI_EXEC
130 131 # Normalize all folders to BASE_ROOT_DIR
132 CI_EXEC rsync --archive --stats --human-readable "${BASE_READ_ONLY_DIR}/" "${BASE_ROOT_DIR}" || echo "Nothing to copy from ${BASE_READ_ONLY_DIR}/"
133 CI_EXEC "${BASE_ROOT_DIR}/ci/test/01_base_install.sh"
134 135 # Fixes permission issues when there is a container UID/GID mismatch with the owner
136 # of the git source code directory.
137 CI_EXEC git config --global --add safe.directory \"*\"
138 139 CI_EXEC mkdir -p "${BINS_SCRATCH_DIR}"
140 141 CI_EXEC "${BASE_ROOT_DIR}/ci/test/03_test_script.sh"
142 143 if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
144 echo "Stop and remove CI container by ID"
145 docker container kill "${CI_CONTAINER_ID}"
146 fi
147