Commit | Line | Data |
---|---|---|
7c5227af | 1 | #!/usr/bin/env python3 |
24fe1f03 | 2 | |
b1a3f243 | 3 | """Find Kconfig symbols that are referenced but not defined.""" |
24fe1f03 | 4 | |
f175ba17 | 5 | # (c) 2014-2016 Valentin Rothberg <valentinrothberg@gmail.com> |
cc641d55 | 6 | # (c) 2014 Stefan Hengelein <stefan.hengelein@fau.de> |
24fe1f03 | 7 | # |
cc641d55 | 8 | # Licensed under the terms of the GNU GPL License version 2 |
24fe1f03 VR |
9 | |
10 | ||
14390e31 | 11 | import argparse |
1b2c8414 | 12 | import difflib |
24fe1f03 VR |
13 | import os |
14 | import re | |
e2042a8a | 15 | import signal |
f175ba17 | 16 | import subprocess |
b1a3f243 | 17 | import sys |
e2042a8a | 18 | from multiprocessing import Pool, cpu_count |
e2042a8a | 19 | from subprocess import Popen, PIPE, STDOUT |
24fe1f03 | 20 | |
cc641d55 VR |
21 | |
22 | # regex expressions | |
24fe1f03 | 23 | OPERATORS = r"&|\(|\)|\||\!" |
cc641d55 VR |
24 | FEATURE = r"(?:\w*[A-Z0-9]\w*){2,}" |
25 | DEF = r"^\s*(?:menu){,1}config\s+(" + FEATURE + r")\s*" | |
24fe1f03 | 26 | EXPR = r"(?:" + OPERATORS + r"|\s|" + FEATURE + r")+" |
0bd38ae3 VR |
27 | DEFAULT = r"default\s+.*?(?:if\s.+){,1}" |
28 | STMT = r"^\s*(?:if|select|depends\s+on|(?:" + DEFAULT + r"))\s+" + EXPR | |
cc641d55 | 29 | SOURCE_FEATURE = r"(?:\W|\b)+[D]{,1}CONFIG_(" + FEATURE + r")" |
24fe1f03 | 30 | |
cc641d55 | 31 | # regex objects |
24fe1f03 | 32 | REGEX_FILE_KCONFIG = re.compile(r".*Kconfig[\.\w+\-]*$") |
e2042a8a | 33 | REGEX_FEATURE = re.compile(r'(?!\B)' + FEATURE + r'(?!\B)') |
cc641d55 VR |
34 | REGEX_SOURCE_FEATURE = re.compile(SOURCE_FEATURE) |
35 | REGEX_KCONFIG_DEF = re.compile(DEF) | |
24fe1f03 VR |
36 | REGEX_KCONFIG_EXPR = re.compile(EXPR) |
37 | REGEX_KCONFIG_STMT = re.compile(STMT) | |
38 | REGEX_KCONFIG_HELP = re.compile(r"^\s+(help|---help---)\s*$") | |
39 | REGEX_FILTER_FEATURES = re.compile(r"[A-Za-z0-9]$") | |
0bd38ae3 | 40 | REGEX_NUMERIC = re.compile(r"0[xX][0-9a-fA-F]+|[0-9]+") |
e2042a8a | 41 | REGEX_QUOTES = re.compile("(\"(.*?)\")") |
24fe1f03 VR |
42 | |
43 | ||
b1a3f243 VR |
44 | def parse_options(): |
45 | """The user interface of this module.""" | |
14390e31 VR |
46 | usage = "Run this tool to detect Kconfig symbols that are referenced but " \ |
47 | "not defined in Kconfig. If no option is specified, " \ | |
48 | "checkkconfigsymbols defaults to check your current tree. " \ | |
49 | "Please note that specifying commits will 'git reset --hard\' " \ | |
50 | "your current tree! You may save uncommitted changes to avoid " \ | |
51 | "losing data." | |
52 | ||
53 | parser = argparse.ArgumentParser(description=usage) | |
54 | ||
55 | parser.add_argument('-c', '--commit', dest='commit', action='store', | |
56 | default="", | |
57 | help="check if the specified commit (hash) introduces " | |
58 | "undefined Kconfig symbols") | |
59 | ||
60 | parser.add_argument('-d', '--diff', dest='diff', action='store', | |
61 | default="", | |
62 | help="diff undefined symbols between two commits " | |
63 | "(e.g., -d commmit1..commit2)") | |
64 | ||
65 | parser.add_argument('-f', '--find', dest='find', action='store_true', | |
66 | default=False, | |
67 | help="find and show commits that may cause symbols to be " | |
68 | "missing (required to run with --diff)") | |
69 | ||
70 | parser.add_argument('-i', '--ignore', dest='ignore', action='store', | |
71 | default="", | |
72 | help="ignore files matching this Python regex " | |
73 | "(e.g., -i '.*defconfig')") | |
74 | ||
75 | parser.add_argument('-s', '--sim', dest='sim', action='store', default="", | |
76 | help="print a list of max. 10 string-similar symbols") | |
77 | ||
78 | parser.add_argument('--force', dest='force', action='store_true', | |
79 | default=False, | |
80 | help="reset current Git tree even when it's dirty") | |
81 | ||
82 | parser.add_argument('--no-color', dest='color', action='store_false', | |
83 | default=True, | |
84 | help="don't print colored output (default when not " | |
85 | "outputting to a terminal)") | |
86 | ||
87 | args = parser.parse_args() | |
88 | ||
89 | if args.commit and args.diff: | |
b1a3f243 VR |
90 | sys.exit("Please specify only one option at once.") |
91 | ||
14390e31 | 92 | if args.diff and not re.match(r"^[\w\-\.]+\.\.[\w\-\.]+$", args.diff): |
b1a3f243 | 93 | sys.exit("Please specify valid input in the following format: " |
38cbfe4f | 94 | "\'commit1..commit2\'") |
b1a3f243 | 95 | |
14390e31 VR |
96 | if args.commit or args.diff: |
97 | if not args.force and tree_is_dirty(): | |
b1a3f243 VR |
98 | sys.exit("The current Git tree is dirty (see 'git status'). " |
99 | "Running this script may\ndelete important data since it " | |
100 | "calls 'git reset --hard' for some performance\nreasons. " | |
101 | " Please run this script in a clean Git tree or pass " | |
102 | "'--force' if you\nwant to ignore this warning and " | |
103 | "continue.") | |
104 | ||
14390e31 VR |
105 | if args.commit: |
106 | args.find = False | |
a42fa92c | 107 | |
14390e31 | 108 | if args.ignore: |
cf132e4a | 109 | try: |
14390e31 | 110 | re.match(args.ignore, "this/is/just/a/test.c") |
cf132e4a VR |
111 | except: |
112 | sys.exit("Please specify a valid Python regex.") | |
113 | ||
14390e31 | 114 | return args |
b1a3f243 VR |
115 | |
116 | ||
24fe1f03 VR |
117 | def main(): |
118 | """Main function of this module.""" | |
14390e31 | 119 | args = parse_options() |
b1a3f243 | 120 | |
4c73c088 | 121 | global color |
14390e31 | 122 | color = args.color and sys.stdout.isatty() |
4c73c088 | 123 | |
14390e31 VR |
124 | if args.sim and not args.commit and not args.diff: |
125 | sims = find_sims(args.sim, args.ignore) | |
1b2c8414 | 126 | if sims: |
7c5227af | 127 | print("%s: %s" % (yel("Similar symbols"), ', '.join(sims))) |
1b2c8414 | 128 | else: |
7c5227af | 129 | print("%s: no similar symbols found" % yel("Similar symbols")) |
1b2c8414 VR |
130 | sys.exit(0) |
131 | ||
132 | # dictionary of (un)defined symbols | |
133 | defined = {} | |
134 | undefined = {} | |
135 | ||
14390e31 | 136 | if args.commit or args.diff: |
b1a3f243 VR |
137 | head = get_head() |
138 | ||
139 | # get commit range | |
140 | commit_a = None | |
141 | commit_b = None | |
14390e31 VR |
142 | if args.commit: |
143 | commit_a = args.commit + "~" | |
144 | commit_b = args.commit | |
145 | elif args.diff: | |
146 | split = args.diff.split("..") | |
b1a3f243 VR |
147 | commit_a = split[0] |
148 | commit_b = split[1] | |
149 | undefined_a = {} | |
150 | undefined_b = {} | |
151 | ||
152 | # get undefined items before the commit | |
153 | execute("git reset --hard %s" % commit_a) | |
14390e31 | 154 | undefined_a, _ = check_symbols(args.ignore) |
b1a3f243 VR |
155 | |
156 | # get undefined items for the commit | |
157 | execute("git reset --hard %s" % commit_b) | |
14390e31 | 158 | undefined_b, defined = check_symbols(args.ignore) |
b1a3f243 VR |
159 | |
160 | # report cases that are present for the commit but not before | |
e9533ae5 | 161 | for feature in sorted(undefined_b): |
b1a3f243 VR |
162 | # feature has not been undefined before |
163 | if not feature in undefined_a: | |
e9533ae5 | 164 | files = sorted(undefined_b.get(feature)) |
1b2c8414 | 165 | undefined[feature] = files |
b1a3f243 VR |
166 | # check if there are new files that reference the undefined feature |
167 | else: | |
e9533ae5 VR |
168 | files = sorted(undefined_b.get(feature) - |
169 | undefined_a.get(feature)) | |
b1a3f243 | 170 | if files: |
1b2c8414 | 171 | undefined[feature] = files |
b1a3f243 VR |
172 | |
173 | # reset to head | |
174 | execute("git reset --hard %s" % head) | |
175 | ||
176 | # default to check the entire tree | |
177 | else: | |
14390e31 | 178 | undefined, defined = check_symbols(args.ignore) |
1b2c8414 VR |
179 | |
180 | # now print the output | |
181 | for feature in sorted(undefined): | |
7c5227af | 182 | print(red(feature)) |
1b2c8414 VR |
183 | |
184 | files = sorted(undefined.get(feature)) | |
7c5227af | 185 | print("%s: %s" % (yel("Referencing files"), ", ".join(files))) |
1b2c8414 | 186 | |
14390e31 | 187 | sims = find_sims(feature, args.ignore, defined) |
1b2c8414 VR |
188 | sims_out = yel("Similar symbols") |
189 | if sims: | |
7c5227af | 190 | print("%s: %s" % (sims_out, ', '.join(sims))) |
1b2c8414 | 191 | else: |
7c5227af | 192 | print("%s: %s" % (sims_out, "no similar symbols found")) |
1b2c8414 | 193 | |
14390e31 | 194 | if args.find: |
7c5227af | 195 | print("%s:" % yel("Commits changing symbol")) |
14390e31 | 196 | commits = find_commits(feature, args.diff) |
1b2c8414 VR |
197 | if commits: |
198 | for commit in commits: | |
199 | commit = commit.split(" ", 1) | |
7c5227af | 200 | print("\t- %s (\"%s\")" % (yel(commit[0]), commit[1])) |
1b2c8414 | 201 | else: |
7c5227af VR |
202 | print("\t- no commit found") |
203 | print() # new line | |
c7455663 VR |
204 | |
205 | ||
206 | def yel(string): | |
207 | """ | |
208 | Color %string yellow. | |
209 | """ | |
4c73c088 | 210 | return "\033[33m%s\033[0m" % string if color else string |
c7455663 VR |
211 | |
212 | ||
213 | def red(string): | |
214 | """ | |
215 | Color %string red. | |
216 | """ | |
4c73c088 | 217 | return "\033[31m%s\033[0m" % string if color else string |
b1a3f243 VR |
218 | |
219 | ||
220 | def execute(cmd): | |
221 | """Execute %cmd and return stdout. Exit in case of error.""" | |
f175ba17 VR |
222 | try: |
223 | cmdlist = cmd.split(" ") | |
7c5227af VR |
224 | stdout = subprocess.check_output(cmdlist, stderr=subprocess.STDOUT, shell=False) |
225 | stdout = stdout.decode(errors='replace') | |
f175ba17 VR |
226 | except subprocess.CalledProcessError as fail: |
227 | exit("Failed to execute %s\n%s" % (cmd, fail)) | |
b1a3f243 VR |
228 | return stdout |
229 | ||
230 | ||
a42fa92c VR |
231 | def find_commits(symbol, diff): |
232 | """Find commits changing %symbol in the given range of %diff.""" | |
233 | commits = execute("git log --pretty=oneline --abbrev-commit -G %s %s" | |
234 | % (symbol, diff)) | |
1b2c8414 | 235 | return [x for x in commits.split("\n") if x] |
a42fa92c VR |
236 | |
237 | ||
b1a3f243 VR |
238 | def tree_is_dirty(): |
239 | """Return true if the current working tree is dirty (i.e., if any file has | |
240 | been added, deleted, modified, renamed or copied but not committed).""" | |
241 | stdout = execute("git status --porcelain") | |
242 | for line in stdout: | |
243 | if re.findall(r"[URMADC]{1}", line[:2]): | |
244 | return True | |
245 | return False | |
246 | ||
247 | ||
248 | def get_head(): | |
249 | """Return commit hash of current HEAD.""" | |
250 | stdout = execute("git rev-parse HEAD") | |
251 | return stdout.strip('\n') | |
252 | ||
253 | ||
e2042a8a VR |
254 | def partition(lst, size): |
255 | """Partition list @lst into eveni-sized lists of size @size.""" | |
7c5227af | 256 | return [lst[i::size] for i in range(size)] |
e2042a8a VR |
257 | |
258 | ||
259 | def init_worker(): | |
260 | """Set signal handler to ignore SIGINT.""" | |
261 | signal.signal(signal.SIGINT, signal.SIG_IGN) | |
262 | ||
263 | ||
1b2c8414 VR |
264 | def find_sims(symbol, ignore, defined = []): |
265 | """Return a list of max. ten Kconfig symbols that are string-similar to | |
266 | @symbol.""" | |
267 | if defined: | |
268 | return sorted(difflib.get_close_matches(symbol, set(defined), 10)) | |
269 | ||
270 | pool = Pool(cpu_count(), init_worker) | |
271 | kfiles = [] | |
272 | for gitfile in get_files(): | |
273 | if REGEX_FILE_KCONFIG.match(gitfile): | |
274 | kfiles.append(gitfile) | |
275 | ||
276 | arglist = [] | |
277 | for part in partition(kfiles, cpu_count()): | |
278 | arglist.append((part, ignore)) | |
279 | ||
280 | for res in pool.map(parse_kconfig_files, arglist): | |
281 | defined.extend(res[0]) | |
282 | ||
283 | return sorted(difflib.get_close_matches(symbol, set(defined), 10)) | |
284 | ||
285 | ||
286 | def get_files(): | |
287 | """Return a list of all files in the current git directory.""" | |
288 | # use 'git ls-files' to get the worklist | |
289 | stdout = execute("git ls-files") | |
290 | if len(stdout) > 0 and stdout[-1] == "\n": | |
291 | stdout = stdout[:-1] | |
292 | ||
293 | files = [] | |
294 | for gitfile in stdout.rsplit("\n"): | |
295 | if ".git" in gitfile or "ChangeLog" in gitfile or \ | |
296 | ".log" in gitfile or os.path.isdir(gitfile) or \ | |
297 | gitfile.startswith("tools/"): | |
298 | continue | |
299 | files.append(gitfile) | |
300 | return files | |
301 | ||
302 | ||
cf132e4a | 303 | def check_symbols(ignore): |
b1a3f243 | 304 | """Find undefined Kconfig symbols and return a dict with the symbol as key |
cf132e4a VR |
305 | and a list of referencing files as value. Files matching %ignore are not |
306 | checked for undefined symbols.""" | |
e2042a8a VR |
307 | pool = Pool(cpu_count(), init_worker) |
308 | try: | |
309 | return check_symbols_helper(pool, ignore) | |
310 | except KeyboardInterrupt: | |
311 | pool.terminate() | |
312 | pool.join() | |
313 | sys.exit(1) | |
314 | ||
315 | ||
316 | def check_symbols_helper(pool, ignore): | |
317 | """Helper method for check_symbols(). Used to catch keyboard interrupts in | |
318 | check_symbols() in order to properly terminate running worker processes.""" | |
24fe1f03 VR |
319 | source_files = [] |
320 | kconfig_files = [] | |
e2042a8a VR |
321 | defined_features = [] |
322 | referenced_features = dict() # {file: [features]} | |
24fe1f03 | 323 | |
1b2c8414 | 324 | for gitfile in get_files(): |
24fe1f03 VR |
325 | if REGEX_FILE_KCONFIG.match(gitfile): |
326 | kconfig_files.append(gitfile) | |
327 | else: | |
e2042a8a VR |
328 | if ignore and not re.match(ignore, gitfile): |
329 | continue | |
330 | # add source files that do not match the ignore pattern | |
24fe1f03 VR |
331 | source_files.append(gitfile) |
332 | ||
e2042a8a VR |
333 | # parse source files |
334 | arglist = partition(source_files, cpu_count()) | |
335 | for res in pool.map(parse_source_files, arglist): | |
336 | referenced_features.update(res) | |
24fe1f03 | 337 | |
e2042a8a VR |
338 | |
339 | # parse kconfig files | |
340 | arglist = [] | |
341 | for part in partition(kconfig_files, cpu_count()): | |
342 | arglist.append((part, ignore)) | |
343 | for res in pool.map(parse_kconfig_files, arglist): | |
344 | defined_features.extend(res[0]) | |
345 | referenced_features.update(res[1]) | |
346 | defined_features = set(defined_features) | |
347 | ||
348 | # inverse mapping of referenced_features to dict(feature: [files]) | |
349 | inv_map = dict() | |
7c5227af | 350 | for _file, features in referenced_features.items(): |
e2042a8a VR |
351 | for feature in features: |
352 | inv_map[feature] = inv_map.get(feature, set()) | |
353 | inv_map[feature].add(_file) | |
354 | referenced_features = inv_map | |
24fe1f03 | 355 | |
b1a3f243 | 356 | undefined = {} # {feature: [files]} |
24fe1f03 | 357 | for feature in sorted(referenced_features): |
cc641d55 VR |
358 | # filter some false positives |
359 | if feature == "FOO" or feature == "BAR" or \ | |
360 | feature == "FOO_BAR" or feature == "XXX": | |
361 | continue | |
24fe1f03 VR |
362 | if feature not in defined_features: |
363 | if feature.endswith("_MODULE"): | |
cc641d55 | 364 | # avoid false positives for kernel modules |
24fe1f03 VR |
365 | if feature[:-len("_MODULE")] in defined_features: |
366 | continue | |
b1a3f243 | 367 | undefined[feature] = referenced_features.get(feature) |
1b2c8414 | 368 | return undefined, defined_features |
24fe1f03 VR |
369 | |
370 | ||
e2042a8a VR |
371 | def parse_source_files(source_files): |
372 | """Parse each source file in @source_files and return dictionary with source | |
373 | files as keys and lists of references Kconfig symbols as values.""" | |
374 | referenced_features = dict() | |
375 | for sfile in source_files: | |
376 | referenced_features[sfile] = parse_source_file(sfile) | |
377 | return referenced_features | |
378 | ||
379 | ||
380 | def parse_source_file(sfile): | |
381 | """Parse @sfile and return a list of referenced Kconfig features.""" | |
24fe1f03 | 382 | lines = [] |
e2042a8a VR |
383 | references = [] |
384 | ||
385 | if not os.path.exists(sfile): | |
386 | return references | |
387 | ||
7c5227af | 388 | with open(sfile, "r", encoding='utf-8', errors='replace') as stream: |
24fe1f03 VR |
389 | lines = stream.readlines() |
390 | ||
391 | for line in lines: | |
392 | if not "CONFIG_" in line: | |
393 | continue | |
394 | features = REGEX_SOURCE_FEATURE.findall(line) | |
395 | for feature in features: | |
396 | if not REGEX_FILTER_FEATURES.search(feature): | |
397 | continue | |
e2042a8a VR |
398 | references.append(feature) |
399 | ||
400 | return references | |
24fe1f03 VR |
401 | |
402 | ||
403 | def get_features_in_line(line): | |
404 | """Return mentioned Kconfig features in @line.""" | |
405 | return REGEX_FEATURE.findall(line) | |
406 | ||
407 | ||
e2042a8a VR |
408 | def parse_kconfig_files(args): |
409 | """Parse kconfig files and return tuple of defined and references Kconfig | |
410 | symbols. Note, @args is a tuple of a list of files and the @ignore | |
411 | pattern.""" | |
412 | kconfig_files = args[0] | |
413 | ignore = args[1] | |
414 | defined_features = [] | |
415 | referenced_features = dict() | |
416 | ||
417 | for kfile in kconfig_files: | |
418 | defined, references = parse_kconfig_file(kfile) | |
419 | defined_features.extend(defined) | |
420 | if ignore and re.match(ignore, kfile): | |
421 | # do not collect references for files that match the ignore pattern | |
422 | continue | |
423 | referenced_features[kfile] = references | |
424 | return (defined_features, referenced_features) | |
425 | ||
426 | ||
427 | def parse_kconfig_file(kfile): | |
24fe1f03 VR |
428 | """Parse @kfile and update feature definitions and references.""" |
429 | lines = [] | |
e2042a8a VR |
430 | defined = [] |
431 | references = [] | |
24fe1f03 VR |
432 | skip = False |
433 | ||
e2042a8a VR |
434 | if not os.path.exists(kfile): |
435 | return defined, references | |
436 | ||
7c5227af | 437 | with open(kfile, "r", encoding='utf-8', errors='replace') as stream: |
24fe1f03 VR |
438 | lines = stream.readlines() |
439 | ||
440 | for i in range(len(lines)): | |
441 | line = lines[i] | |
442 | line = line.strip('\n') | |
cc641d55 | 443 | line = line.split("#")[0] # ignore comments |
24fe1f03 VR |
444 | |
445 | if REGEX_KCONFIG_DEF.match(line): | |
446 | feature_def = REGEX_KCONFIG_DEF.findall(line) | |
e2042a8a | 447 | defined.append(feature_def[0]) |
24fe1f03 VR |
448 | skip = False |
449 | elif REGEX_KCONFIG_HELP.match(line): | |
450 | skip = True | |
451 | elif skip: | |
cc641d55 | 452 | # ignore content of help messages |
24fe1f03 VR |
453 | pass |
454 | elif REGEX_KCONFIG_STMT.match(line): | |
e2042a8a | 455 | line = REGEX_QUOTES.sub("", line) |
24fe1f03 | 456 | features = get_features_in_line(line) |
cc641d55 | 457 | # multi-line statements |
24fe1f03 VR |
458 | while line.endswith("\\"): |
459 | i += 1 | |
460 | line = lines[i] | |
461 | line = line.strip('\n') | |
462 | features.extend(get_features_in_line(line)) | |
463 | for feature in set(features): | |
0bd38ae3 VR |
464 | if REGEX_NUMERIC.match(feature): |
465 | # ignore numeric values | |
466 | continue | |
e2042a8a VR |
467 | references.append(feature) |
468 | ||
469 | return defined, references | |
24fe1f03 VR |
470 | |
471 | ||
472 | if __name__ == "__main__": | |
473 | main() |