get_previous_releases.py raw
1 #!/usr/bin/env python3
2 #
3 # Copyright (c) 2018-present The Bitcoin Core developers
4 # Distributed under the MIT software license, see the accompanying
5 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
6
7 import argparse
8 import contextlib
9 from fnmatch import fnmatch
10 import hashlib
11 import os
12 from pathlib import Path
13 import platform
14 import re
15 import shutil
16 import subprocess
17 import sys
18 import time
19 import zipfile
20
21 sys.path.append(str(Path(__file__).resolve().parent))
22 from download_utils import download_from_url
23
24 TAR = os.getenv('TAR', 'tar')
25
26 SHA256_SUMS = {
27 "0e2819135366f150d9906e294b61dff58fd1996ebd26c2f8e979d6c0b7a79580": {"tag": "v0.14.3", "archive": "bitcoin-0.14.3-aarch64-linux-gnu.tar.gz"},
28 "d86fc90824a85c38b25c8488115178d5785dbc975f5ff674f9f5716bc8ad6e65": {"tag": "v0.14.3", "archive": "bitcoin-0.14.3-arm-linux-gnueabihf.tar.gz"},
29 "1b0a7408c050e3d09a8be8e21e183ef7ee570385dc41216698cc3ab392a484e7": {"tag": "v0.14.3", "archive": "bitcoin-0.14.3-osx64.tar.gz"},
30 "706e0472dbc933ed2757650d54cbcd780fd3829ebf8f609b32780c7eedebdbc9": {"tag": "v0.14.3", "archive": "bitcoin-0.14.3-x86_64-linux-gnu.tar.gz"},
31 "5ea84b6a4c1b03d5f4d1a718fbed215e2fd0e66ca7d59ca6141218d163ce1301": {"tag": "v0.14.3", "archive": "bitcoin-0.14.3-win64.zip"},
32
33 "60c93e3462c303eb080be7cf623f1a7684b37fd47a018ad3848bc23e13c84e1c": {"tag": "v0.20.1", "archive": "bitcoin-0.20.1-aarch64-linux-gnu.tar.gz"},
34 "55b577e0fb306fb429d4be6c9316607753e8543e5946b542d75d876a2f08654c": {"tag": "v0.20.1", "archive": "bitcoin-0.20.1-arm-linux-gnueabihf.tar.gz"},
35 "b9024dde373ea7dad707363e07ec7e265383204127539ae0c234bff3a61da0d1": {"tag": "v0.20.1", "archive": "bitcoin-0.20.1-osx64.tar.gz"},
36 "fa71cb52ee5e0459cbf5248cdec72df27995840c796f58b304607a1ed4c165af": {"tag": "v0.20.1", "archive": "bitcoin-0.20.1-riscv64-linux-gnu.tar.gz"},
37 "376194f06596ecfa40331167c39bc70c355f960280bd2a645fdbf18f66527397": {"tag": "v0.20.1", "archive": "bitcoin-0.20.1-x86_64-linux-gnu.tar.gz"},
38 "e59fba67afce011d32b5d723a3a0be12da1b8a34f5d7966e504520c48d64716d": {"tag": "v0.20.1", "archive": "bitcoin-0.20.1-win64.zip"},
39
40 "43416854330914992bbba2d0e9adf2a6fff4130be9af8ae2ef1186e743d9a3fe": {"tag": "v0.21.0", "archive": "bitcoin-0.21.0-aarch64-linux-gnu.tar.gz"},
41 "f028af308eda45a3c4c90f9332f96b075bf21e3495c945ebce48597151808176": {"tag": "v0.21.0", "archive": "bitcoin-0.21.0-arm-linux-gnueabihf.tar.gz"},
42 "695fb624fa6423f5da4f443b60763dd1d77488bfe5ef63760904a7b54e91298d": {"tag": "v0.21.0", "archive": "bitcoin-0.21.0-osx64.tar.gz"},
43 "f8b2adfeae021a672effbc7bd40d5c48d6b94e53b2dd660f787340bf1a52e4e9": {"tag": "v0.21.0", "archive": "bitcoin-0.21.0-riscv64-linux-gnu.tar.gz"},
44 "da7766775e3f9c98d7a9145429f2be8297c2672fe5b118fd3dc2411fb48e0032": {"tag": "v0.21.0", "archive": "bitcoin-0.21.0-x86_64-linux-gnu.tar.gz"},
45 "1d0052c4ce80227fb6d0bc1c4e673ba21033e219c1f935d25f130ef7f43360d4": {"tag": "v0.21.0", "archive": "bitcoin-0.21.0-win64.zip"},
46
47 "ac718fed08570a81b3587587872ad85a25173afa5f9fbbd0c03ba4d1714cfa3e": {"tag": "v22.0", "archive": "bitcoin-22.0-aarch64-linux-gnu.tar.gz"},
48 "b8713c6c5f03f5258b54e9f436e2ed6d85449aa24c2c9972f91963d413e86311": {"tag": "v22.0", "archive": "bitcoin-22.0-arm-linux-gnueabihf.tar.gz"},
49 "2744d199c3343b2d94faffdfb2c94d75a630ba27301a70e47b0ad30a7e0155e9": {"tag": "v22.0", "archive": "bitcoin-22.0-osx64.tar.gz"},
50 "2cca5f99007d060aca9d8c7cbd035dfe2f040dd8200b210ce32cdf858479f70d": {"tag": "v22.0", "archive": "bitcoin-22.0-powerpc64-linux-gnu.tar.gz"},
51 "91b1e012975c5a363b5b5fcc81b5b7495e86ff703ec8262d4b9afcfec633c30d": {"tag": "v22.0", "archive": "bitcoin-22.0-powerpc64le-linux-gnu.tar.gz"},
52 "9cc3a62c469fe57e11485fdd32c916f10ce7a2899299855a2e479256ff49ff3c": {"tag": "v22.0", "archive": "bitcoin-22.0-riscv64-linux-gnu.tar.gz"},
53 "59ebd25dd82a51638b7a6bb914586201e67db67b919b2a1ff08925a7936d1b16": {"tag": "v22.0", "archive": "bitcoin-22.0-x86_64-linux-gnu.tar.gz"},
54 "9485e4b52ed6cebfe474ab4d7d0c1be6d0bb879ba7246a8239326b2230a77eb1": {"tag": "v22.0", "archive": "bitcoin-22.0-win64.zip"},
55
56 "06f4c78271a77752ba5990d60d81b1751507f77efda1e5981b4e92fd4d9969fb": {"tag": "v23.0", "archive": "bitcoin-23.0-aarch64-linux-gnu.tar.gz"},
57 "952c574366aff76f6d6ad1c9ee45a361d64fa04155e973e926dfe7e26f9703a3": {"tag": "v23.0", "archive": "bitcoin-23.0-arm-linux-gnueabihf.tar.gz"},
58 "7c8bc63731aa872b7b334a8a7d96e33536ad77d49029bad179b09dca32cd77ac": {"tag": "v23.0", "archive": "bitcoin-23.0-arm64-apple-darwin.tar.gz"},
59 "2caa5898399e415f61d9af80a366a3008e5856efa15aaff74b88acf429674c99": {"tag": "v23.0", "archive": "bitcoin-23.0-powerpc64-linux-gnu.tar.gz"},
60 "217dd0469d0f4962d22818c368358575f6a0abcba8804807bb75325eb2f28b19": {"tag": "v23.0", "archive": "bitcoin-23.0-powerpc64le-linux-gnu.tar.gz"},
61 "078f96b1e92895009c798ab827fb3fde5f6719eee886bd0c0e93acab18ea4865": {"tag": "v23.0", "archive": "bitcoin-23.0-riscv64-linux-gnu.tar.gz"},
62 "c816780583009a9dad426dc0c183c89be9da98906e1e2c7ebae91041c1aaaaf3": {"tag": "v23.0", "archive": "bitcoin-23.0-x86_64-apple-darwin.tar.gz"},
63 "2cca490c1f2842884a3c5b0606f179f9f937177da4eadd628e3f7fd7e25d26d0": {"tag": "v23.0", "archive": "bitcoin-23.0-x86_64-linux-gnu.tar.gz"},
64 "004b2e25b21e0f14cbcce6acec37f221447abbb3ea7931c689e508054bfc6cf6": {"tag": "v23.0", "archive": "bitcoin-23.0-win64.zip"},
65
66 "0b48b9e69b30037b41a1e6b78fb7cbcc48c7ad627908c99686e81f3802454609": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-aarch64-linux-gnu.tar.gz"},
67 "37d7660f0277301744e96426bbb001d2206b8d4505385dfdeedf50c09aaaef60": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-arm-linux-gnueabihf.tar.gz"},
68 "90ed59e86bfda1256f4b4cad8cc1dd77ee0efec2492bcb5af61402709288b62c": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-arm64-apple-darwin.tar.gz"},
69 "7590645e8676f8b5fda62dc20174474c4ac8fd0defc83a19ed908ebf2e94dc11": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-powerpc64-linux-gnu.tar.gz"},
70 "79e89a101f23ff87816675b98769cd1ee91059f95c5277f38f48f21a9f7f8509": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-powerpc64le-linux-gnu.tar.gz"},
71 "6b163cef7de4beb07b8cb3347095e0d76a584019b1891135cd1268a1f05b9d88": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-riscv64-linux-gnu.tar.gz"},
72 "e2f751512f3c0f00eb68ba946d9c829e6cf99422a61e8f5e0a7c109c318674d0": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-x86_64-apple-darwin.tar.gz"},
73 "49df6e444515d457ea0b885d66f521f2a26ca92ccf73d5296082e633544253bf": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-x86_64-linux-gnu.tar.gz"},
74 "8784ce304f22c495392d3adfd7fc2c645d093db9bd4d42666c41adf540539fff": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-win64.zip"},
75
76 "3a7bdd959a0b426624f63f394f25e5b7769a5a2f96f8126dcc2ea53f3fa5212b": {"tag": "v25.0", "archive": "bitcoin-25.0-aarch64-linux-gnu.tar.gz"},
77 "e537c8630b05e63242d979c3004f851fd73c2a10b5b4fdbb161788427c7b3c0f": {"tag": "v25.0", "archive": "bitcoin-25.0-arm-linux-gnueabihf.tar.gz"},
78 "3b35075d6c1209743611c705a13575be2668bc069bc6301ce78a2e1e53ebe7cc": {"tag": "v25.0", "archive": "bitcoin-25.0-arm64-apple-darwin.tar.gz"},
79 "0c8e135a6fd297270d3b65196042d761453493a022b5ff7fb847fc911e938214": {"tag": "v25.0", "archive": "bitcoin-25.0-powerpc64-linux-gnu.tar.gz"},
80 "fa8af160782f5adfcea570f72b947073c1663b3e9c3cd0f82b216b609fe47573": {"tag": "v25.0", "archive": "bitcoin-25.0-powerpc64le-linux-gnu.tar.gz"},
81 "fe6e347a66043946920c72c9c4afca301968101e6b82fb90a63d7885ebcceb32": {"tag": "v25.0", "archive": "bitcoin-25.0-riscv64-linux-gnu.tar.gz"},
82 "5708fc639cdfc27347cccfd50db9b73b53647b36fb5f3a4a93537cbe8828c27f": {"tag": "v25.0", "archive": "bitcoin-25.0-x86_64-apple-darwin.tar.gz"},
83 "33930d432593e49d58a9bff4c30078823e9af5d98594d2935862788ce8a20aec": {"tag": "v25.0", "archive": "bitcoin-25.0-x86_64-linux-gnu.tar.gz"},
84 "7154b35ecc8247589070ae739b7c73c4dee4794bea49eb18dc66faed65b819e7": {"tag": "v25.0", "archive": "bitcoin-25.0-win64.zip"},
85
86 "10581b6d4aeb091c08c9e69eb4e4ced000038c009286dc1edb8a876656ccf6df": {"tag": "v28.2", "archive": "bitcoin-28.2-aarch64-linux-gnu.tar.gz"},
87 "c08135c249688438b1ed6f48a2f192ffaae7c7111ffee5cfad59fdae355bb47f": {"tag": "v28.2", "archive": "bitcoin-28.2-arm-linux-gnueabihf.tar.gz"},
88 "c0270ed50effc174f7ff3332dba5183a8693999dac2ba78b37d8c8797b3ea2b2": {"tag": "v28.2", "archive": "bitcoin-28.2-arm64-apple-darwin.tar.gz"},
89 "21c54bc7520e873c8b52c817af257a8a4aee15d81bb2492e5bd51af055ae4469": {"tag": "v28.2", "archive": "bitcoin-28.2-powerpc64-linux-gnu.tar.gz"},
90 "8ad2bf5dd0a7fd04afa9cd2b1309621f8662f00c799f1971cd44ba9eb9d18ef1": {"tag": "v28.2", "archive": "bitcoin-28.2-riscv64-linux-gnu.tar.gz"},
91 "866a4b703a2095301151c17dcc753e19e4dba61ec68d19709ec4f81ff4320103": {"tag": "v28.2", "archive": "bitcoin-28.2-x86_64-apple-darwin.tar.gz"},
92 "98add5f220c01b387343b70edeb6273403fe081e22cd85fda132704cdcaa98aa": {"tag": "v28.2", "archive": "bitcoin-28.2-x86_64-linux-gnu.tar.gz"},
93 "da0869639c323bbf6f264f1829083b9514e10179b90c34b09d8cbcab8a1897e3": {"tag": "v28.2", "archive": "bitcoin-28.2-win64.zip"},
94 }
95
96
97 @contextlib.contextmanager
98 def pushd(new_dir) -> None:
99 previous_dir = os.getcwd()
100 os.chdir(new_dir)
101 try:
102 yield
103 finally:
104 os.chdir(previous_dir)
105
106
107 def download_binary(tag, args) -> int:
108 if Path(tag).is_dir():
109 if not args.remove_dir:
110 print('Using cached {}'.format(tag))
111 return 0
112 shutil.rmtree(tag)
113
114 bin_path = 'bin/bitcoin-core-{}'.format(tag[1:])
115
116 match = re.compile('v(.*)(rc[0-9]+)$').search(tag)
117 if match:
118 bin_path = 'bin/bitcoin-core-{}/test.{}'.format(
119 match.group(1), match.group(2))
120
121 host = args.host
122 if tag < "v23" and host in ["x86_64-apple-darwin", "arm64-apple-darwin"]:
123 host = "osx64"
124
125 archive_format = 'tar.gz'
126 if host == 'win64':
127 archive_format = 'zip'
128
129 archive = f'bitcoin-{tag[1:]}-{host}.{archive_format}'
130 archive_url = f'https://bitcoincore.org/{bin_path}/{archive}'
131
132 try:
133 download_from_url(archive_url, archive)
134 except Exception as e:
135 print(f"\nDownload failed: {e}", file=sys.stderr)
136 print("Retrying download after failure ...", file=sys.stderr)
137 time.sleep(12)
138 try:
139 download_from_url(archive_url, archive)
140 except Exception as e2:
141 print(f"\nDownload failed a second time: {e2}", file=sys.stderr)
142 return 1
143
144 hasher = hashlib.sha256()
145 with open(archive, "rb") as afile:
146 hasher.update(afile.read())
147 archiveHash = hasher.hexdigest()
148
149 if archiveHash not in SHA256_SUMS or SHA256_SUMS[archiveHash]['archive'] != archive:
150 if archive in [v['archive'] for v in SHA256_SUMS.values()]:
151 print(f"Checksum {archiveHash} did not match", file=sys.stderr)
152 else:
153 print("Checksum for given version doesn't exist", file=sys.stderr)
154 return 1
155
156 print("Checksum matched")
157 Path(tag).mkdir()
158
159 # Extract archive
160 if host == 'win64':
161 try:
162 with zipfile.ZipFile(archive, 'r') as zip:
163 zip.extractall(tag)
164 # Remove the top level directory to match tar's --strip-components=1
165 extracted_items = os.listdir(tag)
166 top_level_dir = os.path.join(tag, extracted_items[0])
167 # Move all files & subdirectories up one level
168 for item in os.listdir(top_level_dir):
169 shutil.move(os.path.join(top_level_dir, item), tag)
170 # Remove the now-empty top-level directory
171 os.rmdir(top_level_dir)
172 except Exception as e:
173 print(f"Zip extraction failed: {e}", file=sys.stderr)
174 return 1
175 else:
176 ret = subprocess.run([TAR, '-zxf', archive, '-C', tag,
177 '--strip-components=1',
178 'bitcoin-{tag}'.format(tag=tag[1:])]).returncode
179 if ret != 0:
180 print(f"Failed to extract the {tag} tarball", file=sys.stderr)
181 return ret
182
183 Path(archive).unlink()
184
185 if tag >= "v23" and tag < "v28.2" and args.host == "arm64-apple-darwin":
186 # Starting with v23 there are arm64 binaries for ARM (e.g. M1, M2) mac.
187 # Until v28.2 they had to be signed to run.
188 binary_path = f'{os.getcwd()}/{tag}/bin/'
189
190 for arm_binary in os.listdir(binary_path):
191 # Is it already signed?
192 ret = subprocess.run(
193 ['codesign', '-v', binary_path + arm_binary],
194 stderr=subprocess.DEVNULL, # Suppress expected stderr output
195 ).returncode
196 if ret == 1:
197 # Have to self-sign the binary
198 ret = subprocess.run(
199 ['codesign', '-s', '-', binary_path + arm_binary]
200 ).returncode
201 if ret != 0:
202 print(f"Failed to self-sign {tag} {arm_binary} arm64 binary", file=sys.stderr)
203 return 1
204
205 # Confirm success
206 ret = subprocess.run(
207 ['codesign', '-v', binary_path + arm_binary]
208 ).returncode
209 if ret != 0:
210 print(f"Failed to verify the self-signed {tag} {arm_binary} arm64 binary", file=sys.stderr)
211 return 1
212
213 return 0
214
215
216 def set_host(args) -> int:
217 if platform.system().lower() == 'windows':
218 if platform.machine() != 'AMD64':
219 print('Only 64bit Windows supported', file=sys.stderr)
220 return 1
221 args.host = 'win64'
222 return 0
223 host = os.environ.get('HOST', subprocess.check_output(
224 './depends/config.guess').decode())
225 platforms = {
226 'aarch64-*-linux*': 'aarch64-linux-gnu',
227 'powerpc64le-*-linux-*': 'powerpc64le-linux-gnu',
228 'riscv64-*-linux*': 'riscv64-linux-gnu',
229 'x86_64-*-linux*': 'x86_64-linux-gnu',
230 'x86_64-apple-darwin*': 'x86_64-apple-darwin',
231 'aarch64-apple-darwin*': 'arm64-apple-darwin',
232 }
233 args.host = ''
234 for pattern, target in platforms.items():
235 if fnmatch(host, pattern):
236 args.host = target
237 if not args.host:
238 print('Not sure which binary to download for {}'.format(host), file=sys.stderr)
239 return 1
240 return 0
241
242
243 def main(args) -> int:
244 Path(args.target_dir).mkdir(exist_ok=True, parents=True)
245 print("Releases directory: {}".format(args.target_dir))
246 ret = set_host(args)
247 if ret:
248 return ret
249 with pushd(args.target_dir):
250 for tag in args.tags:
251 ret = download_binary(tag, args)
252 if ret:
253 return ret
254 return 0
255
256
257 if __name__ == '__main__':
258 parser = argparse.ArgumentParser(
259 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
260 epilog='''
261 HOST can be set to any of the `host-platform-triplet`s from
262 depends/README.md for which a release exists.
263 ''',
264 )
265 parser.add_argument('-r', '--remove-dir', action='store_true',
266 help='remove existing directory.')
267 parser.add_argument('-t', '--target-dir', action='store',
268 help='target directory.', default='releases')
269 all_tags = sorted([*set([v['tag'] for v in SHA256_SUMS.values()])])
270 parser.add_argument('tags', nargs='*', default=all_tags,
271 help='release tags. e.g.: v0.18.1 v0.20.0rc2 '
272 '(if not specified, the full list needed for '
273 'backwards compatibility tests will be used)'
274 )
275 args = parser.parse_args()
276 sys.exit(main(args))
277