verify-commits.py raw
1 #!/usr/bin/env python3
2 # Copyright (c) 2018-present The Bitcoin Core developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Verify commits against a trusted keys list."""
6 import argparse
7 import hashlib
8 import logging
9 import os
10 from pathlib import Path
11 import subprocess
12 import sys
13 import time
14
15 GIT = os.getenv('GIT', 'git')
16
17 def tree_sha512sum(commit='HEAD'):
18 """Calculate the Tree-sha512 for the commit.
19
20 This is copied from github-merge.py. See https://github.com/bitcoin-core/bitcoin-maintainer-tools."""
21
22 # request metadata for entire tree, recursively
23 files = []
24 blob_by_name = {}
25 for line in subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', commit]).splitlines():
26 name_sep = line.index(b'\t')
27 metadata = line[:name_sep].split() # perms, 'blob', blobid
28 assert metadata[1] == b'blob'
29 name = line[name_sep + 1:]
30 files.append(name)
31 blob_by_name[name] = metadata[2]
32
33 files.sort()
34 # open connection to git-cat-file in batch mode to request data for all blobs
35 # this is much faster than launching it per file
36 p = subprocess.Popen([GIT, 'cat-file', '--batch'], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
37 overall = hashlib.sha512()
38 for f in files:
39 blob = blob_by_name[f]
40 # request blob
41 p.stdin.write(blob + b'\n')
42 p.stdin.flush()
43 # read header: blob, "blob", size
44 reply = p.stdout.readline().split()
45 assert reply[0] == blob and reply[1] == b'blob'
46 size = int(reply[2])
47 # hash the blob data
48 intern = hashlib.sha512()
49 ptr = 0
50 while ptr < size:
51 bs = min(65536, size - ptr)
52 piece = p.stdout.read(bs)
53 if len(piece) == bs:
54 intern.update(piece)
55 else:
56 raise IOError('Premature EOF reading git cat-file output')
57 ptr += bs
58 dig = intern.hexdigest()
59 assert p.stdout.read(1) == b'\n' # ignore LF that follows blob data
60 # update overall hash with file hash
61 overall.update(dig.encode("utf-8"))
62 overall.update(" ".encode("utf-8"))
63 overall.update(f)
64 overall.update("\n".encode("utf-8"))
65 p.stdin.close()
66 if p.wait():
67 raise IOError('Non-zero return value executing git cat-file')
68 return overall.hexdigest()
69
70 def main():
71
72 # Enable debug logging if running in CI
73 if 'CI' in os.environ and os.environ['CI'].lower() == "true":
74 logging.getLogger().setLevel(logging.DEBUG)
75
76 # Parse arguments
77 parser = argparse.ArgumentParser(usage='%(prog)s [options] [commit id]')
78 parser.add_argument('--disable-tree-check', action='store_false', dest='verify_tree', help='disable SHA-512 tree check')
79 parser.add_argument('--clean-merge', type=float, dest='clean_merge', default=float('inf'), help='Only check clean merge after <NUMBER> days ago (default: %(default)s)', metavar='NUMBER')
80 parser.add_argument('commit', nargs='?', default='HEAD', help='Check clean merge up to commit <commit>')
81 args = parser.parse_args()
82
83 # get directory of this program and read data files
84 dirname = Path(__file__).absolute().parent
85 print(f"Using verify-commits data from {dirname}")
86 verified_root = (dirname / "trusted-git-root").read_text().splitlines()[0]
87 verified_sha512_root = (dirname / "trusted-sha512-root-commit").read_text().splitlines()[0]
88 revsig_allowed = (dirname / "allow-revsig-commits").read_text().splitlines()
89 unclean_merge_allowed = (dirname / "allow-unclean-merge-commits").read_text().splitlines()
90 incorrect_sha512_allowed = (dirname / "allow-incorrect-sha512-commits").read_text().splitlines()
91 trusted_keys = (dirname / "trusted-keys").read_text().splitlines()
92
93 # Set commit and variables
94 current_commit = args.commit
95 if ' ' in current_commit:
96 print("Commit must not contain spaces", file=sys.stderr)
97 sys.exit(1)
98 verify_tree = args.verify_tree
99 no_sha1 = True
100 prev_commit = ""
101 initial_commit = current_commit
102
103 # Iterate through commits
104 while True:
105
106 # Log a message to prevent Travis from timing out
107 logging.debug("verify-commits: [in-progress] processing commit {}".format(current_commit[:8]))
108
109 if current_commit == verified_root:
110 print('There is a valid path from "{}" to {} where all commits are signed!'.format(initial_commit, verified_root))
111 sys.exit(0)
112 else:
113 # Make sure this commit isn't older than trusted roots
114 check_root_older_res = subprocess.run([GIT, "merge-base", "--is-ancestor", verified_root, current_commit])
115 if check_root_older_res.returncode != 0:
116 print(f"\"{current_commit}\" predates the trusted root, stopping!")
117 sys.exit(0)
118
119 if verify_tree:
120 if current_commit == verified_sha512_root:
121 print("All Tree-SHA512s matched up to {}".format(verified_sha512_root), file=sys.stderr)
122 verify_tree = False
123 no_sha1 = False
124 else:
125 # Skip the tree check if we are older than the trusted root
126 check_root_older_res = subprocess.run([GIT, "merge-base", "--is-ancestor", verified_sha512_root, current_commit])
127 if check_root_older_res.returncode != 0:
128 print(f"\"{current_commit}\" predates the trusted SHA512 root, disabling tree verification.")
129 verify_tree = False
130 no_sha1 = False
131
132
133 os.environ['BITCOIN_VERIFY_COMMITS_ALLOW_SHA1'] = "0" if no_sha1 else "1"
134 allow_revsig = current_commit in revsig_allowed
135
136 # Check that the commit (and parents) was signed with a trusted key
137 valid_sig = False
138 verify_res = subprocess.run([GIT, '-c', 'gpg.program={}/gpg.sh'.format(dirname), 'verify-commit', "--raw", current_commit], text=True, capture_output=True)
139 for line in verify_res.stderr.splitlines():
140 if line.startswith("[GNUPG:] VALIDSIG "):
141 key = line.split(" ")[-1]
142 valid_sig = key in trusted_keys
143 elif (line.startswith("[GNUPG:] REVKEYSIG ") or line.startswith("[GNUPG:] EXPKEYSIG ")) and not allow_revsig:
144 valid_sig = False
145 break
146 if not valid_sig:
147 if prev_commit != "":
148 print("No parent of {} was signed with a trusted key!".format(prev_commit), file=sys.stderr)
149 print("Parents are:", file=sys.stderr)
150 parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', prev_commit], text=True).splitlines()[0].split(' ')
151 for parent in parents:
152 subprocess.call([GIT, 'show', '-s', parent], stdout=sys.stderr)
153 else:
154 print("{} was not signed with a trusted key!".format(current_commit), file=sys.stderr)
155 sys.exit(1)
156
157 # Check the Tree-SHA512
158 if (verify_tree or prev_commit == "") and current_commit not in incorrect_sha512_allowed:
159 tree_hash = tree_sha512sum(current_commit)
160 if ("Tree-SHA512: {}".format(tree_hash)) not in subprocess.check_output([GIT, 'show', '-s', '--format=format:%B', current_commit], text=True).splitlines():
161 print("Tree-SHA512 did not match for commit " + current_commit, file=sys.stderr)
162 sys.exit(1)
163
164 # Merge commits should only have two parents
165 parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', current_commit], text=True).splitlines()[0].split(' ')
166 if len(parents) > 2:
167 print("Commit {} is an octopus merge".format(current_commit), file=sys.stderr)
168 sys.exit(1)
169
170 # Check that the merge commit is clean
171 commit_time = int(subprocess.check_output([GIT, 'show', '-s', '--format=format:%ct', current_commit], text=True).splitlines()[0])
172 check_merge = commit_time > time.time() - args.clean_merge * 24 * 60 * 60 # Only check commits in clean_merge days
173 allow_unclean = current_commit in unclean_merge_allowed
174 if len(parents) == 2 and check_merge and not allow_unclean:
175 current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit], text=True).splitlines()[0]
176
177 # This merge-tree functionality requires git >= 2.38. The
178 # --write-tree option was added in order to opt-in to the new
179 # behavior. Older versions of git will not recognize the option and
180 # will instead exit with code 128.
181 try:
182 recreated_tree = subprocess.check_output([GIT, "merge-tree", "--write-tree", parents[0], parents[1]], text=True).splitlines()[0]
183 except subprocess.CalledProcessError as e:
184 if e.returncode == 128:
185 print("git v2.38+ is required for this functionality.", file=sys.stderr)
186 sys.exit(1)
187 else:
188 raise e
189
190 if current_tree != recreated_tree:
191 print("Merge commit {} is not clean".format(current_commit), file=sys.stderr)
192 subprocess.call([GIT, 'diff', recreated_tree, current_tree])
193 sys.exit(1)
194
195 prev_commit = current_commit
196 current_commit = parents[0]
197
198 if __name__ == '__main__':
199 main()
200