1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
|
#!/usr/bin/env python
# SPDX-License-Identifier: GPL-2.0
#
# Copyright (C) Google LLC, 2018
#
# Author: Tom Roeder <tmroeder@google.com>
#
"""A tool for generating compile_commands.json in the Linux kernel."""
import argparse
import json
import logging
import os
import re
_DEFAULT_OUTPUT = 'compile_commands.json'
_DEFAULT_LOG_LEVEL = 'WARNING'
_FILENAME_PATTERN = r'^\..*\.cmd$'
_LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c)$'
_VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
# A kernel build generally has over 2000 entries in its compile_commands.json
# database. If this code finds 300 or fewer, then warn the user that they might
# not have all the .cmd files, and they might need to compile the kernel.
_LOW_COUNT_THRESHOLD = 300
def parse_arguments():
"""Sets up and parses command-line arguments.
Returns:
log_level: A logging level to filter log output.
directory: The directory to search for .cmd files.
output: Where to write the compile-commands JSON file.
"""
usage = 'Creates a compile_commands.json database from kernel .cmd files'
parser = argparse.ArgumentParser(description=usage)
directory_help = ('Path to the kernel source directory to search '
'(defaults to the working directory)')
parser.add_argument('-d', '--directory', type=str, help=directory_help)
output_help = ('The location to write compile_commands.json (defaults to '
'compile_commands.json in the search directory)')
parser.add_argument('-o', '--output', type=str, help=output_help)
log_level_help = ('The level of log messages to produce (one of ' +
', '.join(_VALID_LOG_LEVELS) + '; defaults to ' +
_DEFAULT_LOG_LEVEL + ')')
parser.add_argument(
'--log_level', type=str, default=_DEFAULT_LOG_LEVEL,
help=log_level_help)
args = parser.parse_args()
log_level = args.log_level
if log_level not in _VALID_LOG_LEVELS:
raise ValueError('%s is not a valid log level' % log_level)
directory = args.directory or os.getcwd()
output = args.output or os.path.join(directory, _DEFAULT_OUTPUT)
directory = os.path.abspath(directory)
return log_level, directory, output
def process_line(root_directory, file_directory, command_prefix, relative_path):
"""Extracts information from a .cmd line and creates an entry from it.
Args:
root_directory: The directory that was searched for .cmd files. Usually
used directly in the "directory" entry in compile_commands.json.
file_directory: The path to the directory the .cmd file was found in.
command_prefix: The extracted command line, up to the last element.
relative_path: The .c file from the end of the extracted command.
Usually relative to root_directory, but sometimes relative to
file_directory and sometimes neither.
Returns:
An entry to append to compile_commands.
Raises:
ValueError: Could not find the extracted file based on relative_path and
root_directory or file_directory.
"""
# The .cmd files are intended to be included directly by Make, so they
# escape the pound sign '#', either as '\#' or '$(pound)' (depending on the
# kernel version). The compile_commands.json file is not interepreted
# by Make, so this code replaces the escaped version with '#'.
prefix = command_prefix.replace('\#', '#').replace('$(pound)', '#')
cur_dir = root_directory
expected_path = os.path.join(cur_dir, relative_path)
if not os.path.exists(expected_path):
# Try using file_directory instead. Some of the tools have a different
# style of .cmd file than the kernel.
cur_dir = file_directory
expected_path = os.path.join(cur_dir, relative_path)
if not os.path.exists(expected_path):
raise ValueError('File %s not in %s or %s' %
(relative_path, root_directory, file_directory))
return {
'directory': cur_dir,
'file': relative_path,
'command': prefix + relative_path,
}
def main():
"""Walks through the directory and finds and parses .cmd files."""
log_level, directory, output = parse_arguments()
level = getattr(logging, log_level)
logging.basicConfig(format='%(levelname)s: %(message)s', level=level)
filename_matcher = re.compile(_FILENAME_PATTERN)
line_matcher = re.compile(_LINE_PATTERN)
compile_commands = []
for dirpath, _, filenames in os.walk(directory):
for filename in filenames:
if not filename_matcher.match(filename):
continue
filepath = os.path.join(dirpath, filename)
with open(filepath, 'rt') as f:
for line in f:
result = line_matcher.match(line)
if not result:
continue
try:
entry = process_line(directory, dirpath,
result.group(1), result.group(2))
compile_commands.append(entry)
except ValueError as err:
logging.info('Could not add line from %s: %s',
filepath, err)
with open(output, 'wt') as f:
json.dump(compile_commands, f, indent=2, sort_keys=True)
count = len(compile_commands)
if count < _LOW_COUNT_THRESHOLD:
logging.warning(
'Found %s entries. Have you compiled the kernel?', count)
if __name__ == '__main__':
main()
|
