ci.yml raw
1 # Copyright (c) 2023-present The Limenka developers
2 # Distributed under the MIT software license, see the accompanying
3 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
4
5 name: CI
6 on:
7 # See: https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#pull_request.
8 pull_request:
9 # See: https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#push.
10 push:
11 branches:
12 - '**'
13 tags-ignore:
14 - '**'
15
16 concurrency:
17 group: ${{ github.event_name != 'pull_request' && github.run_id || github.ref }}
18 cancel-in-progress: true
19
20 env:
21 CI_FAILFAST_TEST_LEAVE_DANGLING: 1 # GHA does not care about dangling processes and setting this variable avoids killing the CI script itself on error
22 CIRRUS_CACHE_HOST: http://127.0.0.1:12321/ # When using Cirrus Runners this host can be used by the docker `gha` build cache type.
23 REPO_USE_CIRRUS_RUNNERS: 'limenka/limenka' # Use cirrus runners and cache for this repo, instead of falling back to the slow GHA runners
24
25 jobs:
26 runners:
27 name: 'determine runners'
28 runs-on: ubuntu-latest
29 outputs:
30 use-cirrus-runners: ${{ steps.runners.outputs.use-cirrus-runners }}
31 steps:
32 - id: runners
33 run: |
34 if [[ "${REPO_USE_CIRRUS_RUNNERS}" == "${{ github.repository }}" ]]; then
35 echo "use-cirrus-runners=true" >> "$GITHUB_OUTPUT"
36 echo "::notice title=Runner Selection::Using Cirrus Runners"
37 else
38 echo "use-cirrus-runners=false" >> "$GITHUB_OUTPUT"
39 echo "::notice title=Runner Selection::Using GitHub-hosted runners"
40 fi
41
42 test-each-commit:
43 name: 'test each commit'
44 runs-on: ubuntu-24.04
45 if: github.event_name == 'pull_request' && github.event.pull_request.commits != 1
46 timeout-minutes: 360 # Use maximum time, see https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idtimeout-minutes. Assuming a worst case time of 1 hour per commit, this leads to a --max-count=6 below.
47 env:
48 MAX_COUNT: 6
49 steps:
50 - name: Determine fetch depth
51 run: echo "FETCH_DEPTH=$((${{ github.event.pull_request.commits }} + 2))" >> "$GITHUB_ENV"
52 - uses: actions/checkout@v6
53 with:
54 ref: ${{ github.event.pull_request.head.sha }}
55 fetch-depth: ${{ env.FETCH_DEPTH }}
56 - name: Determine commit range
57 run: |
58 # Checkout HEAD~ and find the test base commit
59 # Checkout HEAD~ because it would be wasteful to rerun tests on the PR
60 # head commit that are already run by other jobs.
61 git checkout HEAD~
62 # Figure out test base commit by listing ancestors of HEAD, excluding
63 # ancestors of the most recent merge commit, limiting the list to the
64 # newest MAX_COUNT ancestors, ordering it from oldest to newest, and
65 # taking the first one.
66 #
67 # If the branch contains up to MAX_COUNT ancestor commits after the
68 # most recent merge commit, all of those commits will be tested. If it
69 # contains more, only the most recent MAX_COUNT commits will be
70 # tested.
71 #
72 # In the command below, the ^@ suffix is used to refer to all parents
73 # of the merge commit as described in:
74 # https://git-scm.com/docs/git-rev-parse#_other_rev_parent_shorthand_notations
75 # and the ^ prefix is used to exclude these parents and all their
76 # ancestors from the rev-list output as described in:
77 # https://git-scm.com/docs/git-rev-list
78 MERGE_BASE=$(git rev-list -n1 --merges HEAD)
79 EXCLUDE_MERGE_BASE_ANCESTORS=
80 # MERGE_BASE can be empty due to limited fetch-depth
81 if test -n "$MERGE_BASE"; then
82 EXCLUDE_MERGE_BASE_ANCESTORS=^${MERGE_BASE}^@
83 fi
84 echo "TEST_BASE=$(git rev-list -n$((${{ env.MAX_COUNT }} + 1)) --reverse HEAD $EXCLUDE_MERGE_BASE_ANCESTORS | head -1)" >> "$GITHUB_ENV"
85 - run: |
86 sudo apt-get update
87 sudo apt-get install clang ccache build-essential cmake pkgconf python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libminiupnpc-dev libzmq3-dev qtbase5-dev qttools5-dev qttools5-dev-tools qtwayland5 libqrencode-dev -y
88 - name: Compile and run tests
89 run: |
90 # Run tests on commits after the last merge commit and before the PR head commit
91 # Use clang++, because it is a bit faster and uses less memory than g++
92 git rebase --exec "echo Running test-one-commit on \$( git log -1 ) && CC=clang CXX=clang++ cmake -B build -DWERROR=ON -DWITH_ZMQ=ON -DBUILD_GUI=ON -DBUILD_BENCH=ON -DBUILD_FUZZ_BINARY=ON -DWITH_BDB=ON -DWARN_INCOMPATIBLE_BDB=OFF -DWITH_MINIUPNPC=ON -DWITH_USDT=ON -DCMAKE_CXX_FLAGS='-Wno-error=unused-member-function' -D RDTS_CONSENT=RUNTIME_CHECK && cmake --build build -j $(nproc) && ctest --output-on-failure --stop-on-failure --test-dir build -j $(nproc) && ./build/test/functional/test_runner.py -j $(( $(nproc) * 2 )) --combinedlogslen=99999999" ${{ env.TEST_BASE }}
93
94 macos-native-arm64:
95 name: ${{ matrix.job-name }}
96 # Use any image to support the xcode-select below, but hardcode version to avoid silent upgrades (and breaks).
97 # See: https://github.com/actions/runner-images#available-images.
98 runs-on: macos-14
99
100 # When a contributor maintains a fork of the repo, any pull request they make
101 # to their own fork, or to the main repository, will trigger two CI runs:
102 # one for the branch push and one for the pull request.
103 # This can be avoided by setting SKIP_BRANCH_PUSH=true as a custom env variable
104 # in Github repository settings.
105 if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }}
106
107 timeout-minutes: 120
108
109 strategy:
110 fail-fast: false
111 matrix:
112 job-type: [standard, fuzz]
113 include:
114 - job-type: standard
115 file-env: './ci/test/00_setup_env_mac_native.sh'
116 job-name: 'macOS 14 native, arm64, no depends, sqlite only, gui'
117 - job-type: fuzz
118 file-env: './ci/test/00_setup_env_mac_native_fuzz.sh'
119 job-name: 'macOS 14 native, arm64, fuzz'
120
121 env:
122 DANGER_RUN_CI_ON_HOST: 1
123 BASE_ROOT_DIR: ${{ github.workspace }}
124
125 steps:
126 - &CHECKOUT
127 name: Checkout
128 uses: actions/checkout@v6
129 with:
130 # Ensure the latest merged pull request state is used, even on re-runs.
131 ref: &CHECKOUT_REF_TMPL ${{ github.event_name == 'pull_request' && github.ref || '' }}
132
133 - name: Clang version
134 run: |
135 # Use the earliest Xcode supported by the version of macOS denoted in
136 # doc/release-notes-empty-template.md and providing at least the
137 # minimum clang version denoted in doc/dependencies.md.
138 # See: https://developer.apple.com/documentation/xcode-release-notes/xcode-15-release-notes
139 sudo xcode-select --switch /Applications/Xcode_15.0.app
140 clang --version
141
142 - name: Install Homebrew packages
143 env:
144 HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1
145 run: |
146 # A workaround for "The `brew link` step did not complete successfully" error.
147 brew install --quiet python@3 || brew link --overwrite python@3
148 brew install --quiet coreutils ninja pkgconf gnu-getopt ccache boost libevent miniupnpc zeromq qt@5 qrencode imagemagick libicns librsvg
149
150 - name: Set Ccache directory
151 run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV"
152
153 - name: Restore Ccache cache
154 id: ccache-cache
155 uses: actions/cache/restore@v4
156 with:
157 path: ${{ env.CCACHE_DIR }}
158 key: ${{ github.job }}-${{ matrix.job-type }}-ccache-${{ github.run_id }}
159 restore-keys: ${{ github.job }}-${{ matrix.job-type }}-ccache-
160
161 - name: CI script
162 run: ./ci/test_run_all.sh
163 env:
164 FILE_ENV: ${{ matrix.file-env }}
165
166 - name: Save Ccache cache
167 uses: actions/cache/save@v5
168 if: github.event_name != 'pull_request' && steps.ccache-cache.outputs.cache-hit != 'true'
169 with:
170 path: ${{ env.CCACHE_DIR }}
171 # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
172 key: ${{ github.job }}-${{ matrix.job-type }}-ccache-${{ github.run_id }}
173
174 win64-native:
175 name: ${{ matrix.job-name }}
176 # Use latest image, but hardcode version to avoid silent upgrades (and breaks).
177 # See: https://github.com/actions/runner-images#available-images.
178 runs-on: windows-2022
179
180 if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }}
181
182 env:
183 PYTHONUTF8: 1
184 TEST_RUNNER_TIMEOUT_FACTOR: 40
185
186 strategy:
187 fail-fast: false
188 matrix:
189 job-type: [standard, fuzz]
190 include:
191 - job-type: standard
192 generate-options: '-DBUILD_GUI=OFF -DWITH_BDB=ON -DWITH_MINIUPNPC=ON -DWITH_ZMQ=ON -DBUILD_BENCH=ON -DWERROR=ON'
193 job-name: 'Win64 native, VS 2022'
194 - job-type: fuzz
195 generate-options: '-DVCPKG_MANIFEST_NO_DEFAULT_FEATURES=ON -DVCPKG_MANIFEST_FEATURES="sqlite" -DBUILD_GUI=OFF -DBUILD_FOR_FUZZING=ON -DWERROR=ON'
196 job-name: 'Win64 native fuzz, VS 2022'
197
198 steps:
199 - *CHECKOUT
200
201 - name: Set up VS Developer Prompt
202 shell: pwsh -Command "$PSVersionTable; $PSNativeCommandUseErrorActionPreference = $true; $ErrorActionPreference = 'Stop'; & '{0}'"
203 run: |
204 $vswherePath = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe"
205 $installationPath = & $vswherePath -latest -property installationPath
206 & "${env:COMSPEC}" /s /c "`"$installationPath\Common7\Tools\vsdevcmd.bat`" -arch=x64 -no_logo && set" | foreach-object {
207 $name, $value = $_ -split '=', 2
208 echo "$name=$value" >> $env:GITHUB_ENV
209 }
210
211 - name: Get tool information
212 run: |
213 cmake -version | Tee-Object -FilePath "cmake_version"
214 Write-Output "---"
215 msbuild -version | Tee-Object -FilePath "msbuild_version"
216 $env:VCToolsVersion | Tee-Object -FilePath "toolset_version"
217 py -3 --version
218 Write-Host "PowerShell version $($PSVersionTable.PSVersion.ToString())"
219
220 - name: Using vcpkg with MSBuild
221 shell: bash
222 run: |
223 echo "set(VCPKG_BUILD_TYPE release)" >> "${VCPKG_INSTALLATION_ROOT}/triplets/x64-windows.cmake"
224 echo "set(VCPKG_BUILD_TYPE release)" >> "${VCPKG_INSTALLATION_ROOT}/triplets/x64-windows-static.cmake"
225 # Workaround for libevent, which requires CMake 3.1 but is incompatible with CMake >= 4.0.
226 sed -i '1s/^/set(ENV{CMAKE_POLICY_VERSION_MINIMUM} 3.5)\n/' "${VCPKG_INSTALLATION_ROOT}/scripts/ports.cmake"
227
228 - name: vcpkg tools cache
229 uses: actions/cache@v5
230 with:
231 path: C:/vcpkg/downloads/tools
232 key: ${{ github.job }}-vcpkg-tools
233
234 - name: Restore vcpkg binary cache
235 uses: actions/cache/restore@v5
236 id: vcpkg-binary-cache
237 with:
238 path: ~/AppData/Local/vcpkg/archives
239 key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('cmake_version', 'msbuild_version', 'toolset_version', 'vcpkg.json') }}
240
241 - name: Generate build system
242 run: |
243 cmake -B build --preset vs2022-static -DCMAKE_TOOLCHAIN_FILE="$env:VCPKG_INSTALLATION_ROOT\scripts\buildsystems\vcpkg.cmake" -D RDTS_CONSENT=RUNTIME_CHECK ${{ matrix.generate-options }}
244
245 - name: Save vcpkg binary cache
246 uses: actions/cache/save@v5
247 if: github.event_name != 'pull_request' && steps.vcpkg-binary-cache.outputs.cache-hit != 'true' && matrix.job-type == 'standard'
248 with:
249 path: ~/AppData/Local/vcpkg/archives
250 key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('cmake_version', 'msbuild_version', 'toolset_version', 'vcpkg.json') }}
251
252 - name: Build
253 working-directory: build
254 run: |
255 cmake --build . -j $env:NUMBER_OF_PROCESSORS --config Release
256
257 - name: Run test suite
258 if: matrix.job-type == 'standard'
259 working-directory: build
260 run: |
261 ctest --output-on-failure --stop-on-failure -j $env:NUMBER_OF_PROCESSORS -C Release
262
263 - name: Run functional tests
264 if: matrix.job-type == 'standard'
265 working-directory: build
266 env:
267 LIMENKAD: '${{ github.workspace }}\build\bin\Release\limenkad.exe'
268 LIMENKACLI: '${{ github.workspace }}\build\bin\Release\limenka-cli.exe'
269 LIMENKAUTIL: '${{ github.workspace }}\build\bin\Release\limenka-util.exe'
270 LIMENKAWALLET: '${{ github.workspace }}\build\bin\Release\limenka-wallet.exe'
271 TEST_RUNNER_EXTRA: ${{ github.event_name != 'pull_request' && '--extended' || '' }}
272 shell: cmd
273 run: py -3 test\functional\test_runner.py --jobs %NUMBER_OF_PROCESSORS% --ci --quiet --tmpdirprefix=%RUNNER_TEMP% --combinedlogslen=99999999 --timeout-factor=%TEST_RUNNER_TIMEOUT_FACTOR% %TEST_RUNNER_EXTRA%
274
275 - name: Clone corpora
276 if: matrix.job-type == 'fuzz'
277 run: |
278 git clone --depth=1 https://github.com/limenka/qa-assets "$env:RUNNER_TEMP\qa-assets"
279 Set-Location "$env:RUNNER_TEMP\qa-assets"
280 Write-Host "Using qa-assets repo from commit ..."
281 git log -1
282
283 - name: Run fuzz tests
284 if: matrix.job-type == 'fuzz'
285 working-directory: build
286 env:
287 LIMENKAFUZZ: '${{ github.workspace }}\build\bin\Release\fuzz.exe'
288 shell: cmd
289 run: |
290 py -3 test\fuzz\test_runner.py --par %NUMBER_OF_PROCESSORS% --loglevel DEBUG %RUNNER_TEMP%\qa-assets\fuzz_corpora
291
292 ci-matrix:
293 name: ${{ matrix.name }}
294 needs: runners
295 runs-on: ${{ needs.runners.outputs.use-cirrus-runners == 'true' && matrix.cirrus-runner || matrix.fallback-runner }}
296 if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }}
297 timeout-minutes: ${{ matrix.timeout-minutes }}
298
299 env:
300 DANGER_CI_ON_HOST_FOLDERS: 1
301 FILE_ENV: ${{ matrix.file-env }}
302
303 strategy:
304 fail-fast: false
305 matrix:
306 include:
307 - name: '32 bit ARM, unit tests, no functional tests'
308 cirrus-runner: 'ubuntu-24.04-arm' # Cirrus' Arm runners are Apple (with virtual Linux aarch64), which doesn't support 32-bit mode
309 fallback-runner: 'ubuntu-24.04-arm'
310 timeout-minutes: 120
311 file-env: './ci/test/00_setup_env_arm.sh'
312
313 - name: 'win64 Cross'
314 cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm'
315 fallback-runner: 'ubuntu-24.04'
316 timeout-minutes: 120
317 file-env: './ci/test/00_setup_env_win64.sh'
318
319 - name: 'ASan + LSan + UBSan + integer, no depends, USDT'
320 cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' # has to match container in ci/test/00_setup_env_native_asan.sh for tracing tools
321 fallback-runner: 'ubuntu-24.04'
322 timeout-minutes: 120
323 file-env: './ci/test/00_setup_env_native_asan.sh'
324
325 - name: 'macOS-cross, gui, no tests'
326 cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm'
327 fallback-runner: 'ubuntu-24.04'
328 timeout-minutes: 120
329 file-env: './ci/test/00_setup_env_mac_cross.sh'
330
331 - name: 'No wallet, liblimenkakernel'
332 cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm'
333 fallback-runner: 'ubuntu-24.04'
334 timeout-minutes: 120
335 file-env: './ci/test/00_setup_env_native_nowallet_liblimenkakernel.sh'
336
337 - name: 'i686, multiprocess, DEBUG'
338 cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md'
339 fallback-runner: 'ubuntu-24.04'
340 timeout-minutes: 120
341 file-env: './ci/test/00_setup_env_i686_multiprocess.sh'
342
343 - name: 'fuzzer,address,undefined,integer, no depends'
344 cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg'
345 fallback-runner: 'ubuntu-24.04'
346 timeout-minutes: 240
347 file-env: './ci/test/00_setup_env_native_fuzz.sh'
348
349 - name: 'previous releases, depends DEBUG'
350 cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md'
351 fallback-runner: 'ubuntu-24.04'
352 timeout-minutes: 120
353 file-env: './ci/test/00_setup_env_native_previous_releases.sh'
354
355 - name: 'CentOS, depends, gui'
356 cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg'
357 fallback-runner: 'ubuntu-24.04'
358 timeout-minutes: 120
359 file-env: './ci/test/00_setup_env_native_centos.sh'
360
361 - name: 'tidy'
362 cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md'
363 fallback-runner: 'ubuntu-24.04'
364 timeout-minutes: 120
365 file-env: './ci/test/00_setup_env_native_tidy.sh'
366
367 - name: 'TSan, depends, no gui'
368 cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md'
369 fallback-runner: 'ubuntu-24.04'
370 timeout-minutes: 120
371 file-env: './ci/test/00_setup_env_native_tsan.sh'
372
373 - name: 'MSan, depends'
374 cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg'
375 fallback-runner: 'ubuntu-24.04'
376 timeout-minutes: 120
377 file-env: './ci/test/00_setup_env_native_msan.sh'
378
379 steps:
380 - *CHECKOUT
381
382 - name: Configure environment
383 uses: ./.github/actions/configure-environment
384
385 - name: Restore caches
386 id: restore-cache
387 uses: ./.github/actions/restore-caches
388
389 - name: Configure Docker
390 uses: ./.github/actions/configure-docker
391 with:
392 use-cirrus: ${{ needs.runners.outputs.use-cirrus-runners }}
393
394 - name: Enable bpfcc script
395 if: ${{ env.CONTAINER_NAME == 'ci_native_asan' }}
396 # In the image build step, no external environment variables are available,
397 # so any settings will need to be written to the settings env file:
398 run: sed -i "s|\${INSTALL_BCC_TRACING_TOOLS}|true|g" ./ci/test/00_setup_env_native_asan.sh
399
400 - name: Set mmap_rnd_bits
401 if: ${{ env.CONTAINER_NAME == 'ci_native_tsan' || env.CONTAINER_NAME == 'ci_native_msan' }}
402 # Prevents crashes due to high ASLR entropy
403 run: sudo sysctl -w vm.mmap_rnd_bits=28
404
405 - name: CI script
406 run: ./ci/test_run_all.sh
407
408 - name: Save caches
409 uses: ./.github/actions/save-caches
410
411 lint:
412 name: 'lint'
413 needs: runners
414 runs-on: ${{ needs.runners.outputs.use-cirrus-runners == 'true' && 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-xs' || 'ubuntu-24.04' }}
415 if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }}
416 timeout-minutes: 20
417 env:
418 CONTAINER_NAME: "limenka-linter"
419 steps:
420 - name: Checkout
421 uses: actions/checkout@v6
422 with:
423 ref: *CHECKOUT_REF_TMPL
424 fetch-depth: 0
425
426 - name: Configure Docker
427 uses: ./.github/actions/configure-docker
428 with:
429 use-cirrus: ${{ needs.runners.outputs.use-cirrus-runners }}
430
431 - name: CI script
432 run: |
433 set -o xtrace
434 docker buildx build -t "$CONTAINER_NAME" $DOCKER_BUILD_CACHE_ARG --file "./ci/lint_imagefile" .
435 CIRRUS_PR_FLAG=""
436 if [ "${{ github.event_name }}" = "pull_request" ]; then
437 CIRRUS_PR_FLAG="-e CIRRUS_PR=1"
438 fi
439 docker run --rm $CIRRUS_PR_FLAG -v "$(pwd)":/limenka "$CONTAINER_NAME"
440