1 #!/usr/bin/env python3
2 # Copyright (c) 2021-present The Bitcoin Core developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 6 """
7 This checks that all files in the repository have correct filenames and permissions
8 """
9 10 import os
11 import re
12 import sys
13 from subprocess import check_output
14 from typing import Optional, NoReturn
15 16 CMD_TOP_LEVEL = ["git", "rev-parse", "--show-toplevel"]
17 CMD_ALL_FILES = ["git", "ls-files", "-z", "--full-name", "--stage"]
18 CMD_SHEBANG_FILES = ["git", "grep", "--full-name", "--line-number", "-I", "^#!"]
19 20 ALL_SOURCE_FILENAMES_REGEXP = r"^.*\.(cpp|h|py|sh)$"
21 ALLOWED_FILENAME_REGEXP = "^[a-zA-Z0-9/_.@][a-zA-Z0-9/_.@-]*$"
22 ALLOWED_SOURCE_FILENAME_REGEXP = "^[a-z0-9_./-]+$"
23 ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP = (
24 "^src/(secp256k1/|minisketch/|test/fuzz/FuzzedDataProvider.h)"
25 )
26 ALLOWED_PERMISSION_NON_EXECUTABLES = 0o644
27 ALLOWED_PERMISSION_EXECUTABLES = 0o755
28 ALLOWED_EXECUTABLE_SHEBANG = {
29 # https://github.com/dylanaraps/pure-bash-bible#shebang:
30 # `#!/bin/bash` assumes it is always installed to /bin/ which can cause issues;
31 # `#!/usr/bin/env bash` searches the user's PATH to find the bash binary.
32 "py": [b"#!/usr/bin/env python3"],
33 "sh": [b"#!/usr/bin/env bash", b"#!/bin/sh"],
34 }
35 36 37 class FileMeta(object):
38 def __init__(self, file_spec: str):
39 '''Parse a `git ls files --stage` output line.'''
40 # 100755 5a150d5f8031fcd75e80a4dd9843afa33655f579 0 ci/test/00_setup_env.sh
41 meta, self.file_path = file_spec.split('\t', 2)
42 meta = meta.split()
43 # The octal file permission of the file. Internally, git only
44 # keeps an 'executable' bit, so this will always be 0o644 or 0o755.
45 self.permissions = int(meta[0], 8) & 0o7777
46 # We don't currently care about the other fields
47 48 @property
49 def extension(self) -> Optional[str]:
50 """
51 Returns the file extension for a given filename string.
52 eg:
53 'ci/lint_run_all.sh' -> 'sh'
54 'ci/retry/retry' -> None
55 'contrib/devtools/split-debug.sh.in' -> 'in'
56 """
57 return str(os.path.splitext(self.file_path)[1].strip(".") or None)
58 59 @property
60 def full_extension(self) -> Optional[str]:
61 """
62 Returns the full file extension for a given filename string.
63 eg:
64 'ci/lint_run_all.sh' -> 'sh'
65 'ci/retry/retry' -> None
66 'contrib/devtools/split-debug.sh.in' -> 'sh.in'
67 """
68 filename_parts = self.file_path.split(os.extsep, 1)
69 try:
70 return filename_parts[1]
71 except IndexError:
72 return None
73 74 75 def get_git_file_metadata() -> dict[str, FileMeta]:
76 '''
77 Return a dictionary mapping the name of all files in the repository to git tree metadata.
78 '''
79 files_raw = check_output(CMD_ALL_FILES, text=True).rstrip("\0").split("\0")
80 files = {}
81 for file_spec in files_raw:
82 meta = FileMeta(file_spec)
83 files[meta.file_path] = meta
84 return files
85 86 def check_all_filenames(files) -> int:
87 """
88 Checks every file in the repository against an allowed regexp to make sure only lowercase or uppercase
89 alphanumerics (a-zA-Z0-9), underscores (_), hyphens (-), at (@) and dots (.) are used in repository filenames.
90 """
91 filenames = files.keys()
92 filename_regex = re.compile(ALLOWED_FILENAME_REGEXP)
93 failed_tests = 0
94 for filename in filenames:
95 if not filename_regex.match(filename):
96 print(
97 f"""File {repr(filename)} does not match the allowed filename regexp ('{ALLOWED_FILENAME_REGEXP}')."""
98 )
99 failed_tests += 1
100 return failed_tests
101 102 103 def check_source_filenames(files) -> int:
104 """
105 Checks only source files (*.cpp, *.h, *.py, *.sh) against a stricter allowed regexp to make sure only lowercase
106 alphanumerics (a-z0-9), underscores (_), hyphens (-) and dots (.) are used in source code filenames.
107 108 Additionally there is an exception regexp for directories or files which are excepted from matching this regexp.
109 """
110 filenames = [filename for filename in files.keys() if re.match(ALL_SOURCE_FILENAMES_REGEXP, filename, re.IGNORECASE)]
111 filename_regex = re.compile(ALLOWED_SOURCE_FILENAME_REGEXP)
112 filename_exception_regex = re.compile(ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP)
113 failed_tests = 0
114 for filename in filenames:
115 if not filename_regex.match(filename) and not filename_exception_regex.match(filename):
116 print(
117 f"""File {repr(filename)} does not match the allowed source filename regexp ('{ALLOWED_SOURCE_FILENAME_REGEXP}'), or the exception regexp ({ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP})."""
118 )
119 failed_tests += 1
120 return failed_tests
121 122 123 def check_all_file_permissions(files) -> int:
124 """
125 Checks all files in the repository match an allowed executable or non-executable file permission octal.
126 127 Additionally checks that for executable files, the file contains a shebang line
128 """
129 failed_tests = 0
130 for filename, file_meta in files.items():
131 if file_meta.permissions == ALLOWED_PERMISSION_EXECUTABLES:
132 with open(filename, "rb") as f:
133 shebang = f.readline().rstrip(b"\n")
134 135 # For any file with executable permissions the first line must contain a shebang
136 if not shebang.startswith(b"#!"):
137 print(
138 f"""File "{filename}" has permission {ALLOWED_PERMISSION_EXECUTABLES:03o} (executable) and is thus expected to contain a shebang '#!'. Add shebang or do "chmod {ALLOWED_PERMISSION_NON_EXECUTABLES:03o} {filename}" to make it non-executable."""
139 )
140 failed_tests += 1
141 142 # For certain file extensions that have been defined, we also check that the shebang conforms to a specific
143 # allowable set of shebangs
144 if file_meta.extension in ALLOWED_EXECUTABLE_SHEBANG.keys():
145 if shebang not in ALLOWED_EXECUTABLE_SHEBANG[file_meta.extension]:
146 print(
147 f"""File "{filename}" is missing expected shebang """
148 + " or ".join(
149 [
150 x.decode("utf-8")
151 for x in ALLOWED_EXECUTABLE_SHEBANG[file_meta.extension]
152 ]
153 )
154 )
155 failed_tests += 1
156 157 elif file_meta.permissions == ALLOWED_PERMISSION_NON_EXECUTABLES:
158 continue
159 else:
160 print(
161 f"""File "{filename}" has unexpected permission {file_meta.permissions:03o}. Do "chmod {ALLOWED_PERMISSION_NON_EXECUTABLES:03o} {filename}" (if non-executable) or "chmod {ALLOWED_PERMISSION_EXECUTABLES:03o} {filename}" (if executable)."""
162 )
163 failed_tests += 1
164 165 return failed_tests
166 167 168 def check_shebang_file_permissions(files_meta) -> int:
169 """
170 Checks every file that contains a shebang line to ensure it has an executable permission
171 """
172 filenames = check_output(CMD_SHEBANG_FILES, text=True).strip().split("\n")
173 174 # The git grep command we use returns files which contain a shebang on any line within the file
175 # so we need to filter the list to only files with the shebang on the first line
176 filenames = [filename.split(":1:")[0] for filename in filenames if ":1:" in filename]
177 178 failed_tests = 0
179 for filename in filenames:
180 file_meta = files_meta[filename]
181 if file_meta.permissions != ALLOWED_PERMISSION_EXECUTABLES:
182 # These file types are typically expected to be sourced and not executed directly
183 if file_meta.full_extension in ["bash", "init", "openrc", "sh.in"]:
184 continue
185 186 # *.py files which don't contain an `if __name__ == '__main__'` are not expected to be executed directly
187 if file_meta.extension == "py":
188 with open(filename, "r") as f:
189 file_data = f.read()
190 if not re.search("""if __name__ == ['"]__main__['"]:""", file_data):
191 continue
192 193 print(
194 f"""File "{filename}" contains a shebang line, but has the file permission {file_meta.permissions:03o} instead of the expected executable permission {ALLOWED_PERMISSION_EXECUTABLES:03o}. Do "chmod {ALLOWED_PERMISSION_EXECUTABLES:03o} {filename}" (or remove the shebang line)."""
195 )
196 failed_tests += 1
197 return failed_tests
198 199 200 def main() -> NoReturn:
201 root_dir = check_output(CMD_TOP_LEVEL, text=True).strip()
202 os.chdir(root_dir)
203 204 files = get_git_file_metadata()
205 206 failed_tests = 0
207 failed_tests += check_all_filenames(files)
208 failed_tests += check_source_filenames(files)
209 failed_tests += check_all_file_permissions(files)
210 failed_tests += check_shebang_file_permissions(files)
211 212 if failed_tests:
213 print(
214 f"ERROR: There were {failed_tests} failed tests in the lint-files.py lint test. Please resolve the above errors."
215 )
216 sys.exit(1)
217 else:
218 sys.exit(0)
219 220 221 if __name__ == "__main__":
222 main()
223