copyright_header.py raw
1 #!/usr/bin/env python3
2 # Copyright (c) 2016-2022 The Limenka developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5
6 import re
7 import fnmatch
8 import sys
9 import subprocess
10 import datetime
11 import os
12
13 ################################################################################
14 # file filtering
15 ################################################################################
16
17 EXCLUDE = [
18 # auto generated:
19 'src/qt/limenkastrings.cpp',
20 'src/chainparamsseeds.h',
21 # other external copyrights:
22 'src/test/fuzz/FuzzedDataProvider.h',
23 'src/tinyformat.h',
24 'src/bench/nanobench.h',
25 'test/functional/test_framework/bignum.py',
26 # python init:
27 '*__init__.py',
28 ]
29 EXCLUDE_COMPILED = re.compile('|'.join([fnmatch.translate(m) for m in EXCLUDE]))
30
31 EXCLUDE_DIRS = [
32 # git subtrees
33 "src/crypto/ctaes/",
34 "src/leveldb/",
35 "src/minisketch",
36 "src/secp256k1/",
37 "src/crc32c/",
38 ]
39
40 INCLUDE = ['*.h', '*.cpp', '*.cc', '*.c', '*.mm', '*.py', '*.sh', '*.bash-completion']
41 INCLUDE_COMPILED = re.compile('|'.join([fnmatch.translate(m) for m in INCLUDE]))
42
43 def applies_to_file(filename):
44 for excluded_dir in EXCLUDE_DIRS:
45 if filename.startswith(excluded_dir):
46 return False
47 return ((EXCLUDE_COMPILED.match(filename) is None) and
48 (INCLUDE_COMPILED.match(filename) is not None))
49
50 ################################################################################
51 # obtain list of files in repo according to INCLUDE and EXCLUDE
52 ################################################################################
53
54 GIT_LS_CMD = 'git ls-files --full-name'.split(' ')
55 GIT_TOPLEVEL_CMD = 'git rev-parse --show-toplevel'.split(' ')
56
57 def call_git_ls(base_directory):
58 out = subprocess.check_output([*GIT_LS_CMD, base_directory])
59 return [f for f in out.decode("utf-8").split('\n') if f != '']
60
61 def call_git_toplevel():
62 "Returns the absolute path to the project root"
63 return subprocess.check_output(GIT_TOPLEVEL_CMD).strip().decode("utf-8")
64
65 def get_filenames_to_examine(base_directory):
66 "Returns an array of absolute paths to any project files in the base_directory that pass the include/exclude filters"
67 root = call_git_toplevel()
68 filenames = call_git_ls(base_directory)
69 return sorted([os.path.join(root, filename) for filename in filenames if
70 applies_to_file(filename)])
71
72 ################################################################################
73 # define and compile regexes for the patterns we are looking for
74 ################################################################################
75
76
77 COPYRIGHT_WITH_C = r'Copyright \(c\)'
78 COPYRIGHT_WITHOUT_C = 'Copyright'
79 ANY_COPYRIGHT_STYLE = '(%s|%s)' % (COPYRIGHT_WITH_C, COPYRIGHT_WITHOUT_C)
80
81 YEAR = "20[0-9][0-9]"
82 YEAR_RANGE = '(%s)(-%s)?' % (YEAR, YEAR)
83 YEAR_LIST = '(%s)(, %s)+' % (YEAR, YEAR)
84 ANY_YEAR_STYLE = '(%s|%s)' % (YEAR_RANGE, YEAR_LIST)
85 ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE = ("%s %s" % (ANY_COPYRIGHT_STYLE,
86 ANY_YEAR_STYLE))
87
88 ANY_COPYRIGHT_COMPILED = re.compile(ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE)
89
90 def compile_copyright_regex(copyright_style, year_style, name):
91 return re.compile(r'%s %s,? %s( +\*)?\n' % (copyright_style, year_style, name))
92
93 EXPECTED_HOLDER_NAMES = [
94 r"Satoshi Nakamoto",
95 r"The Limenka developers",
96 r"BitPay Inc\.",
97 r"University of Illinois at Urbana-Champaign\.",
98 r"Pieter Wuille",
99 r"Wladimir J\. van der Laan",
100 r"Jeff Garzik",
101 r"Jan-Klaas Kollhof",
102 r"ArtForz -- public domain half-a-node",
103 r"Intel Corporation ?",
104 r"The Zcash developers",
105 r"Jeremy Rubin",
106 ]
107
108 DOMINANT_STYLE_COMPILED = {}
109 YEAR_LIST_STYLE_COMPILED = {}
110 WITHOUT_C_STYLE_COMPILED = {}
111
112 for holder_name in EXPECTED_HOLDER_NAMES:
113 DOMINANT_STYLE_COMPILED[holder_name] = (
114 compile_copyright_regex(COPYRIGHT_WITH_C, YEAR_RANGE, holder_name))
115 YEAR_LIST_STYLE_COMPILED[holder_name] = (
116 compile_copyright_regex(COPYRIGHT_WITH_C, YEAR_LIST, holder_name))
117 WITHOUT_C_STYLE_COMPILED[holder_name] = (
118 compile_copyright_regex(COPYRIGHT_WITHOUT_C, ANY_YEAR_STYLE,
119 holder_name))
120
121 ################################################################################
122 # search file contents for copyright message of particular category
123 ################################################################################
124
125 def get_count_of_copyrights_of_any_style_any_holder(contents):
126 return len(ANY_COPYRIGHT_COMPILED.findall(contents))
127
128 def file_has_dominant_style_copyright_for_holder(contents, holder_name):
129 match = DOMINANT_STYLE_COMPILED[holder_name].search(contents)
130 return match is not None
131
132 def file_has_year_list_style_copyright_for_holder(contents, holder_name):
133 match = YEAR_LIST_STYLE_COMPILED[holder_name].search(contents)
134 return match is not None
135
136 def file_has_without_c_style_copyright_for_holder(contents, holder_name):
137 match = WITHOUT_C_STYLE_COMPILED[holder_name].search(contents)
138 return match is not None
139
140 ################################################################################
141 # get file info
142 ################################################################################
143
144 def read_file(filename):
145 return open(filename, 'r', encoding="utf8").read()
146
147 def gather_file_info(filename):
148 info = {}
149 info['filename'] = filename
150 c = read_file(filename)
151 info['contents'] = c
152
153 info['all_copyrights'] = get_count_of_copyrights_of_any_style_any_holder(c)
154
155 info['classified_copyrights'] = 0
156 info['dominant_style'] = {}
157 info['year_list_style'] = {}
158 info['without_c_style'] = {}
159 for holder_name in EXPECTED_HOLDER_NAMES:
160 has_dominant_style = (
161 file_has_dominant_style_copyright_for_holder(c, holder_name))
162 has_year_list_style = (
163 file_has_year_list_style_copyright_for_holder(c, holder_name))
164 has_without_c_style = (
165 file_has_without_c_style_copyright_for_holder(c, holder_name))
166 info['dominant_style'][holder_name] = has_dominant_style
167 info['year_list_style'][holder_name] = has_year_list_style
168 info['without_c_style'][holder_name] = has_without_c_style
169 if has_dominant_style or has_year_list_style or has_without_c_style:
170 info['classified_copyrights'] = info['classified_copyrights'] + 1
171 return info
172
173 ################################################################################
174 # report execution
175 ################################################################################
176
177 SEPARATOR = '-'.join(['' for _ in range(80)])
178
179 def print_filenames(filenames, verbose):
180 if not verbose:
181 return
182 for filename in filenames:
183 print("\t%s" % filename)
184
185 def print_report(file_infos, verbose):
186 print(SEPARATOR)
187 examined = [i['filename'] for i in file_infos]
188 print("%d files examined according to INCLUDE and EXCLUDE fnmatch rules" %
189 len(examined))
190 print_filenames(examined, verbose)
191
192 print(SEPARATOR)
193 print('')
194 zero_copyrights = [i['filename'] for i in file_infos if
195 i['all_copyrights'] == 0]
196 print("%4d with zero copyrights" % len(zero_copyrights))
197 print_filenames(zero_copyrights, verbose)
198 one_copyright = [i['filename'] for i in file_infos if
199 i['all_copyrights'] == 1]
200 print("%4d with one copyright" % len(one_copyright))
201 print_filenames(one_copyright, verbose)
202 two_copyrights = [i['filename'] for i in file_infos if
203 i['all_copyrights'] == 2]
204 print("%4d with two copyrights" % len(two_copyrights))
205 print_filenames(two_copyrights, verbose)
206 three_copyrights = [i['filename'] for i in file_infos if
207 i['all_copyrights'] == 3]
208 print("%4d with three copyrights" % len(three_copyrights))
209 print_filenames(three_copyrights, verbose)
210 four_or_more_copyrights = [i['filename'] for i in file_infos if
211 i['all_copyrights'] >= 4]
212 print("%4d with four or more copyrights" % len(four_or_more_copyrights))
213 print_filenames(four_or_more_copyrights, verbose)
214 print('')
215 print(SEPARATOR)
216 print('Copyrights with dominant style:\ne.g. "Copyright (c)" and '
217 '"<year>" or "<startYear>-<endYear>":\n')
218 for holder_name in EXPECTED_HOLDER_NAMES:
219 dominant_style = [i['filename'] for i in file_infos if
220 i['dominant_style'][holder_name]]
221 if len(dominant_style) > 0:
222 print("%4d with '%s'" % (len(dominant_style),
223 holder_name.replace('\n', '\\n')))
224 print_filenames(dominant_style, verbose)
225 print('')
226 print(SEPARATOR)
227 print('Copyrights with year list style:\ne.g. "Copyright (c)" and '
228 '"<year1>, <year2>, ...":\n')
229 for holder_name in EXPECTED_HOLDER_NAMES:
230 year_list_style = [i['filename'] for i in file_infos if
231 i['year_list_style'][holder_name]]
232 if len(year_list_style) > 0:
233 print("%4d with '%s'" % (len(year_list_style),
234 holder_name.replace('\n', '\\n')))
235 print_filenames(year_list_style, verbose)
236 print('')
237 print(SEPARATOR)
238 print('Copyrights with no "(c)" style:\ne.g. "Copyright" and "<year>" or '
239 '"<startYear>-<endYear>":\n')
240 for holder_name in EXPECTED_HOLDER_NAMES:
241 without_c_style = [i['filename'] for i in file_infos if
242 i['without_c_style'][holder_name]]
243 if len(without_c_style) > 0:
244 print("%4d with '%s'" % (len(without_c_style),
245 holder_name.replace('\n', '\\n')))
246 print_filenames(without_c_style, verbose)
247
248 print('')
249 print(SEPARATOR)
250
251 unclassified_copyrights = [i['filename'] for i in file_infos if
252 i['classified_copyrights'] < i['all_copyrights']]
253 print("%d with unexpected copyright holder names" %
254 len(unclassified_copyrights))
255 print_filenames(unclassified_copyrights, verbose)
256 print(SEPARATOR)
257
258 def exec_report(base_directory, verbose):
259 filenames = get_filenames_to_examine(base_directory)
260 file_infos = [gather_file_info(f) for f in filenames]
261 print_report(file_infos, verbose)
262
263 ################################################################################
264 # report cmd
265 ################################################################################
266
267 REPORT_USAGE = """
268 Produces a report of all copyright header notices found inside the source files
269 of a repository.
270
271 Usage:
272 $ ./copyright_header.py report <base_directory> [verbose]
273
274 Arguments:
275 <base_directory> - The base directory of a limenka source code repository.
276 [verbose] - Includes a list of every file of each subcategory in the report.
277 """
278
279 def report_cmd(argv):
280 if len(argv) == 2:
281 sys.exit(REPORT_USAGE)
282
283 base_directory = argv[2]
284 if not os.path.exists(base_directory):
285 sys.exit("*** bad <base_directory>: %s" % base_directory)
286
287 if len(argv) == 3:
288 verbose = False
289 elif argv[3] == 'verbose':
290 verbose = True
291 else:
292 sys.exit("*** unknown argument: %s" % argv[2])
293
294 exec_report(base_directory, verbose)
295
296 ################################################################################
297 # query git for year of last change
298 ################################################################################
299
300 GIT_LOG_CMD = "git log --pretty=format:%%ai %s"
301
302 def call_git_log(filename):
303 out = subprocess.check_output((GIT_LOG_CMD % filename).split(' '))
304 return out.decode("utf-8").split('\n')
305
306 def get_git_change_years(filename):
307 git_log_lines = call_git_log(filename)
308 if len(git_log_lines) == 0:
309 return [datetime.date.today().year]
310 # timestamp is in ISO 8601 format. e.g. "2016-09-05 14:25:32 -0600"
311 return [line.split(' ')[0].split('-')[0] for line in git_log_lines]
312
313 def get_most_recent_git_change_year(filename):
314 return max(get_git_change_years(filename))
315
316 ################################################################################
317 # read and write to file
318 ################################################################################
319
320 def read_file_lines(filename):
321 with open(filename, 'r', encoding="utf8") as f:
322 file_lines = f.readlines()
323 return file_lines
324
325 def write_file_lines(filename, file_lines):
326 with open(filename, 'w', encoding="utf8") as f:
327 f.write(''.join(file_lines))
328
329 ################################################################################
330 # update header years execution
331 ################################################################################
332
333 COPYRIGHT = r'Copyright \(c\)'
334 YEAR = "20[0-9][0-9]"
335 YEAR_RANGE = '(%s)(-%s)?' % (YEAR, YEAR)
336 HOLDER = 'The Limenka developers'
337 UPDATEABLE_LINE_COMPILED = re.compile(' '.join([COPYRIGHT, YEAR_RANGE, HOLDER]))
338
339 def get_updatable_copyright_line(file_lines):
340 index = 0
341 for line in file_lines:
342 if UPDATEABLE_LINE_COMPILED.search(line) is not None:
343 return index, line
344 index = index + 1
345 return None, None
346
347 def parse_year_range(year_range):
348 year_split = year_range.split('-')
349 start_year = year_split[0]
350 if len(year_split) == 1:
351 return start_year, start_year
352 return start_year, year_split[1]
353
354 def year_range_to_str(start_year, end_year):
355 if start_year == end_year:
356 return start_year
357 return "%s-%s" % (start_year, end_year)
358
359 def create_updated_copyright_line(line, last_git_change_year):
360 copyright_splitter = 'Copyright (c) '
361 copyright_split = line.split(copyright_splitter)
362 # Preserve characters on line that are ahead of the start of the copyright
363 # notice - they are part of the comment block and vary from file-to-file.
364 before_copyright = copyright_split[0]
365 after_copyright = copyright_split[1]
366
367 space_split = after_copyright.split(' ')
368 year_range = space_split[0]
369 start_year, end_year = parse_year_range(year_range)
370 if end_year >= last_git_change_year:
371 return line
372 return (before_copyright + copyright_splitter +
373 year_range_to_str(start_year, last_git_change_year) + ' ' +
374 ' '.join(space_split[1:]))
375
376 def update_updatable_copyright(filename):
377 file_lines = read_file_lines(filename)
378 index, line = get_updatable_copyright_line(file_lines)
379 if not line:
380 print_file_action_message(filename, "No updatable copyright.")
381 return
382 last_git_change_year = get_most_recent_git_change_year(filename)
383 new_line = create_updated_copyright_line(line, last_git_change_year)
384 if line == new_line:
385 print_file_action_message(filename, "Copyright up-to-date.")
386 return
387 file_lines[index] = new_line
388 write_file_lines(filename, file_lines)
389 print_file_action_message(filename,
390 "Copyright updated! -> %s" % last_git_change_year)
391
392 def exec_update_header_year(base_directory):
393 for filename in get_filenames_to_examine(base_directory):
394 update_updatable_copyright(filename)
395
396 ################################################################################
397 # update cmd
398 ################################################################################
399
400 UPDATE_USAGE = """
401 Updates all the copyright headers of "The Limenka developers" which were
402 changed in a year more recent than is listed. For example:
403
404 // Copyright (c) <firstYear>-<lastYear> The Limenka developers
405
406 will be updated to:
407
408 // Copyright (c) <firstYear>-<lastModifiedYear> The Limenka developers
409
410 where <lastModifiedYear> is obtained from the 'git log' history.
411
412 This subcommand also handles copyright headers that have only a single year. In those cases:
413
414 // Copyright (c) <year> The Limenka developers
415
416 will be updated to:
417
418 // Copyright (c) <year>-<lastModifiedYear> The Limenka developers
419
420 where the update is appropriate.
421
422 Usage:
423 $ ./copyright_header.py update <base_directory>
424
425 Arguments:
426 <base_directory> - The base directory of a limenka source code repository.
427 """
428
429 def print_file_action_message(filename, action):
430 print("%-52s %s" % (filename, action))
431
432 def update_cmd(argv):
433 if len(argv) != 3:
434 sys.exit(UPDATE_USAGE)
435
436 base_directory = argv[2]
437 if not os.path.exists(base_directory):
438 sys.exit("*** bad base_directory: %s" % base_directory)
439 exec_update_header_year(base_directory)
440
441 ################################################################################
442 # inserted copyright header format
443 ################################################################################
444
445 def get_header_lines(header, start_year, end_year):
446 lines = header.split('\n')[1:-1]
447 lines[0] = lines[0] % year_range_to_str(start_year, end_year)
448 return [line + '\n' for line in lines]
449
450 CPP_HEADER = '''
451 // Copyright (c) %s The Limenka developers
452 // Distributed under the MIT software license, see the accompanying
453 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
454 '''
455
456 def get_cpp_header_lines_to_insert(start_year, end_year):
457 return reversed(get_header_lines(CPP_HEADER, start_year, end_year))
458
459 SCRIPT_HEADER = '''
460 # Copyright (c) %s The Limenka developers
461 # Distributed under the MIT software license, see the accompanying
462 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
463 '''
464
465 def get_script_header_lines_to_insert(start_year, end_year):
466 return reversed(get_header_lines(SCRIPT_HEADER, start_year, end_year))
467
468 ################################################################################
469 # query git for year of last change
470 ################################################################################
471
472 def get_git_change_year_range(filename):
473 years = get_git_change_years(filename)
474 return min(years), max(years)
475
476 ################################################################################
477 # check for existing core copyright
478 ################################################################################
479
480 def file_already_has_core_copyright(file_lines):
481 index, _ = get_updatable_copyright_line(file_lines)
482 return index is not None
483
484 ################################################################################
485 # insert header execution
486 ################################################################################
487
488 def file_has_hashbang(file_lines):
489 if len(file_lines) < 1:
490 return False
491 if len(file_lines[0]) <= 2:
492 return False
493 return file_lines[0][:2] == '#!'
494
495 def insert_script_header(filename, file_lines, start_year, end_year):
496 if file_has_hashbang(file_lines):
497 insert_idx = 1
498 else:
499 insert_idx = 0
500 header_lines = get_script_header_lines_to_insert(start_year, end_year)
501 for line in header_lines:
502 file_lines.insert(insert_idx, line)
503 write_file_lines(filename, file_lines)
504
505 def insert_cpp_header(filename, file_lines, start_year, end_year):
506 file_lines.insert(0, '\n')
507 header_lines = get_cpp_header_lines_to_insert(start_year, end_year)
508 for line in header_lines:
509 file_lines.insert(0, line)
510 write_file_lines(filename, file_lines)
511
512 def exec_insert_header(filename, style):
513 file_lines = read_file_lines(filename)
514 if file_already_has_core_copyright(file_lines):
515 sys.exit('*** %s already has a copyright by The Limenka developers'
516 % (filename))
517 start_year, end_year = get_git_change_year_range(filename)
518 if style in ['python', 'shell']:
519 insert_script_header(filename, file_lines, start_year, end_year)
520 else:
521 insert_cpp_header(filename, file_lines, start_year, end_year)
522
523 ################################################################################
524 # insert cmd
525 ################################################################################
526
527 INSERT_USAGE = """
528 Inserts a copyright header for "The Limenka developers" at the top of the
529 file in either Python or C++ style as determined by the file extension. If the
530 file is a Python file and it has a '#!' starting the first line, the header is
531 inserted in the line below it.
532
533 The copyright dates will be set to be:
534
535 "<year_introduced>-<current_year>"
536
537 where <year_introduced> is according to the 'git log' history. If
538 <year_introduced> is equal to <current_year>, the date will be set to be:
539
540 "<current_year>"
541
542 If the file already has a copyright for "The Limenka developers", the
543 script will exit.
544
545 Usage:
546 $ ./copyright_header.py insert <file>
547
548 Arguments:
549 <file> - A source file in the limenka repository.
550 """
551
552 def insert_cmd(argv):
553 if len(argv) != 3:
554 sys.exit(INSERT_USAGE)
555
556 filename = argv[2]
557 if not os.path.isfile(filename):
558 sys.exit("*** bad filename: %s" % filename)
559 _, extension = os.path.splitext(filename)
560 if extension not in ['.h', '.cpp', '.cc', '.c', '.py', '.sh']:
561 sys.exit("*** cannot insert for file extension %s" % extension)
562
563 if extension == '.py':
564 style = 'python'
565 elif extension == '.sh':
566 style = 'shell'
567 else:
568 style = 'cpp'
569 exec_insert_header(filename, style)
570
571 ################################################################################
572 # UI
573 ################################################################################
574
575 USAGE = """
576 copyright_header.py - utilities for managing copyright headers of 'The Limenka
577 Core developers' in repository source files.
578
579 Usage:
580 $ ./copyright_header <subcommand>
581
582 Subcommands:
583 report
584 update
585 insert
586
587 To see subcommand usage, run them without arguments.
588 """
589
590 SUBCOMMANDS = ['report', 'update', 'insert']
591
592 if __name__ == "__main__":
593 if len(sys.argv) == 1:
594 sys.exit(USAGE)
595 subcommand = sys.argv[1]
596 if subcommand not in SUBCOMMANDS:
597 sys.exit(USAGE)
598 if subcommand == 'report':
599 report_cmd(sys.argv)
600 elif subcommand == 'update':
601 update_cmd(sys.argv)
602 elif subcommand == 'insert':
603 insert_cmd(sys.argv)
604