1 #!/usr/bin/env python3
2 # Copyright (c) 2021-2022 The Limenka developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 6 """
7 This checks that all files in the repository have correct filenames and permissions
8 """
9 10 import os
11 import re
12 import sys
13 from subprocess import check_output
14 from typing import Optional, NoReturn
15 16 CMD_TOP_LEVEL = ["git", "rev-parse", "--show-toplevel"]
17 CMD_ALL_FILES = ["git", "ls-files", "-z", "--full-name", "--stage"]
18 CMD_SHEBANG_FILES = ["git", "grep", "--full-name", "--line-number", "-I", "^#!"]
19 20 ALL_SOURCE_FILENAMES_REGEXP = r"^.*\.(cpp|h|py|sh)$"
21 ALLOWED_FILENAME_REGEXP = "^[a-zA-Z0-9/_.@][a-zA-Z0-9/_.@-]*$"
22 ALLOWED_SOURCE_FILENAME_REGEXP = "^[a-z0-9_./-]+$"
23 ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP = (
24 "^src/(secp256k1/|minisketch/|test/fuzz/FuzzedDataProvider.h)"
25 )
26 ALLOWED_PERMISSION_NON_EXECUTABLES = 0o644
27 ALLOWED_PERMISSION_EXECUTABLES = 0o755
28 ALLOWED_EXECUTABLE_SHEBANG = {
29 "py": [b"#!/usr/bin/env python3"],
30 "sh": [b"#!/usr/bin/env bash", b"#!/bin/sh"],
31 }
32 33 34 class FileMeta(object):
35 def __init__(self, file_spec: str):
36 '''Parse a `git ls files --stage` output line.'''
37 # 100755 5a150d5f8031fcd75e80a4dd9843afa33655f579 0 ci/test/00_setup_env.sh
38 meta, self.file_path = file_spec.split('\t', 2)
39 meta = meta.split()
40 # The octal file permission of the file. Internally, git only
41 # keeps an 'executable' bit, so this will always be 0o644 or 0o755.
42 self.permissions = int(meta[0], 8) & 0o7777
43 # We don't currently care about the other fields
44 45 @property
46 def extension(self) -> Optional[str]:
47 """
48 Returns the file extension for a given filename string.
49 eg:
50 'ci/lint_run_all.sh' -> 'sh'
51 'ci/retry/retry' -> None
52 'contrib/devtools/split-debug.sh.in' -> 'in'
53 """
54 return str(os.path.splitext(self.file_path)[1].strip(".") or None)
55 56 @property
57 def full_extension(self) -> Optional[str]:
58 """
59 Returns the full file extension for a given filename string.
60 eg:
61 'ci/lint_run_all.sh' -> 'sh'
62 'ci/retry/retry' -> None
63 'contrib/devtools/split-debug.sh.in' -> 'sh.in'
64 """
65 filename_parts = self.file_path.split(os.extsep, 1)
66 try:
67 return filename_parts[1]
68 except IndexError:
69 return None
70 71 72 def get_git_file_metadata() -> dict[str, FileMeta]:
73 '''
74 Return a dictionary mapping the name of all files in the repository to git tree metadata.
75 '''
76 files_raw = check_output(CMD_ALL_FILES).decode("utf8").rstrip("\0").split("\0")
77 files = {}
78 for file_spec in files_raw:
79 meta = FileMeta(file_spec)
80 files[meta.file_path] = meta
81 return files
82 83 def check_all_filenames(files) -> int:
84 """
85 Checks every file in the repository against an allowed regexp to make sure only lowercase or uppercase
86 alphanumerics (a-zA-Z0-9), underscores (_), hyphens (-), at (@) and dots (.) are used in repository filenames.
87 """
88 filenames = files.keys()
89 filename_regex = re.compile(ALLOWED_FILENAME_REGEXP)
90 failed_tests = 0
91 for filename in filenames:
92 if not filename_regex.match(filename):
93 print(
94 f"""File {repr(filename)} does not match the allowed filename regexp ('{ALLOWED_FILENAME_REGEXP}')."""
95 )
96 failed_tests += 1
97 return failed_tests
98 99 100 def check_source_filenames(files) -> int:
101 """
102 Checks only source files (*.cpp, *.h, *.py, *.sh) against a stricter allowed regexp to make sure only lowercase
103 alphanumerics (a-z0-9), underscores (_), hyphens (-) and dots (.) are used in source code filenames.
104 105 Additionally there is an exception regexp for directories or files which are excepted from matching this regexp.
106 """
107 filenames = [filename for filename in files.keys() if re.match(ALL_SOURCE_FILENAMES_REGEXP, filename, re.IGNORECASE)]
108 filename_regex = re.compile(ALLOWED_SOURCE_FILENAME_REGEXP)
109 filename_exception_regex = re.compile(ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP)
110 failed_tests = 0
111 for filename in filenames:
112 if not filename_regex.match(filename) and not filename_exception_regex.match(filename):
113 print(
114 f"""File {repr(filename)} does not match the allowed source filename regexp ('{ALLOWED_SOURCE_FILENAME_REGEXP}'), or the exception regexp ({ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP})."""
115 )
116 failed_tests += 1
117 return failed_tests
118 119 120 def check_all_file_permissions(files) -> int:
121 """
122 Checks all files in the repository match an allowed executable or non-executable file permission octal.
123 124 Additionally checks that for executable files, the file contains a shebang line
125 """
126 failed_tests = 0
127 for filename, file_meta in files.items():
128 if file_meta.permissions == ALLOWED_PERMISSION_EXECUTABLES:
129 with open(filename, "rb") as f:
130 shebang = f.readline().rstrip(b"\n")
131 132 # For any file with executable permissions the first line must contain a shebang
133 if not shebang.startswith(b"#!"):
134 print(
135 f"""File "{filename}" has permission {ALLOWED_PERMISSION_EXECUTABLES:03o} (executable) and is thus expected to contain a shebang '#!'. Add shebang or do "chmod {ALLOWED_PERMISSION_NON_EXECUTABLES:03o} {filename}" to make it non-executable."""
136 )
137 failed_tests += 1
138 139 # For certain file extensions that have been defined, we also check that the shebang conforms to a specific
140 # allowable set of shebangs
141 if file_meta.extension in ALLOWED_EXECUTABLE_SHEBANG.keys():
142 if shebang not in ALLOWED_EXECUTABLE_SHEBANG[file_meta.extension]:
143 print(
144 f"""File "{filename}" is missing expected shebang """
145 + " or ".join(
146 [
147 x.decode("utf-8")
148 for x in ALLOWED_EXECUTABLE_SHEBANG[file_meta.extension]
149 ]
150 )
151 )
152 failed_tests += 1
153 154 elif file_meta.permissions == ALLOWED_PERMISSION_NON_EXECUTABLES:
155 continue
156 else:
157 print(
158 f"""File "{filename}" has unexpected permission {file_meta.permissions:03o}. Do "chmod {ALLOWED_PERMISSION_NON_EXECUTABLES:03o} {filename}" (if non-executable) or "chmod {ALLOWED_PERMISSION_EXECUTABLES:03o} {filename}" (if executable)."""
159 )
160 failed_tests += 1
161 162 return failed_tests
163 164 165 def check_shebang_file_permissions(files_meta) -> int:
166 """
167 Checks every file that contains a shebang line to ensure it has an executable permission
168 """
169 filenames = check_output(CMD_SHEBANG_FILES).decode("utf8").strip().split("\n")
170 171 # The git grep command we use returns files which contain a shebang on any line within the file
172 # so we need to filter the list to only files with the shebang on the first line
173 filenames = [filename.split(":1:")[0] for filename in filenames if ":1:" in filename]
174 175 failed_tests = 0
176 for filename in filenames:
177 file_meta = files_meta[filename]
178 if file_meta.permissions != ALLOWED_PERMISSION_EXECUTABLES:
179 # These file types are typically expected to be sourced and not executed directly
180 if file_meta.full_extension in ["bash", "init", "openrc", "sh.in"]:
181 continue
182 183 # *.py files which don't contain an `if __name__ == '__main__'` are not expected to be executed directly
184 if file_meta.extension == "py":
185 with open(filename, "r", encoding="utf8") as f:
186 file_data = f.read()
187 if not re.search("""if __name__ == ['"]__main__['"]:""", file_data):
188 continue
189 190 print(
191 f"""File "{filename}" contains a shebang line, but has the file permission {file_meta.permissions:03o} instead of the expected executable permission {ALLOWED_PERMISSION_EXECUTABLES:03o}. Do "chmod {ALLOWED_PERMISSION_EXECUTABLES:03o} {filename}" (or remove the shebang line)."""
192 )
193 failed_tests += 1
194 return failed_tests
195 196 197 def main() -> NoReturn:
198 root_dir = check_output(CMD_TOP_LEVEL).decode("utf8").strip()
199 os.chdir(root_dir)
200 201 files = get_git_file_metadata()
202 203 failed_tests = 0
204 failed_tests += check_all_filenames(files)
205 failed_tests += check_source_filenames(files)
206 failed_tests += check_all_file_permissions(files)
207 failed_tests += check_shebang_file_permissions(files)
208 209 if failed_tests:
210 print(
211 f"ERROR: There were {failed_tests} failed tests in the lint-files.py lint test. Please resolve the above errors."
212 )
213 sys.exit(1)
214 else:
215 sys.exit(0)
216 217 218 if __name__ == "__main__":
219 main()
220