| 1 |
import os.path |
| 2 |
import re |
| 3 |
|
| 4 |
from . import common as _common |
| 5 |
|
| 6 |
# The following C files must not built with Py_BUILD_CORE. |
| 7 |
FILES_WITHOUT_INTERNAL_CAPI = frozenset(( |
| 8 |
# Modules/ |
| 9 |
'_testcapimodule.c', |
| 10 |
'_testlimitedcapi.c', |
| 11 |
'_testclinic_limited.c', |
| 12 |
'xxlimited.c', |
| 13 |
'xxlimited_35.c', |
| 14 |
)) |
| 15 |
|
| 16 |
# C files in the fhe following directories must not be built with |
| 17 |
# Py_BUILD_CORE. |
| 18 |
DIRS_WITHOUT_INTERNAL_CAPI = frozenset(( |
| 19 |
'_testcapi', # Modules/_testcapi/ |
| 20 |
'_testlimitedcapi', # Modules/_testlimitedcapi/ |
| 21 |
)) |
| 22 |
|
| 23 |
TOOL = 'gcc' |
| 24 |
|
| 25 |
META_FILES = { |
| 26 |
'<built-in>', |
| 27 |
'<command-line>', |
| 28 |
} |
| 29 |
|
| 30 |
# https://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html |
| 31 |
# flags: |
| 32 |
# 1 start of a new file |
| 33 |
# 2 returning to a file (after including another) |
| 34 |
# 3 following text comes from a system header file |
| 35 |
# 4 following text treated wrapped in implicit extern "C" block |
| 36 |
LINE_MARKER_RE = re.compile(r'^# (\d+) "([^"]+)"((?: [1234])*)$') |
| 37 |
PREPROC_DIRECTIVE_RE = re.compile(r'^\s*#\s*(\w+)\b.*') |
| 38 |
COMPILER_DIRECTIVE_RE = re.compile(r''' |
| 39 |
^ |
| 40 |
(.*?) # <before> |
| 41 |
(__\w+__) # <directive> |
| 42 |
\s* |
| 43 |
[(] [(] |
| 44 |
( |
| 45 |
[^()]* |
| 46 |
(?: |
| 47 |
[(] |
| 48 |
[^()]* |
| 49 |
[)] |
| 50 |
[^()]* |
| 51 |
)* |
| 52 |
) # <args> |
| 53 |
( [)] [)] ) # <closed> |
| 54 |
''', re.VERBOSE) |
| 55 |
|
| 56 |
POST_ARGS = ( |
| 57 |
'-pthread', |
| 58 |
'-std=c99', |
| 59 |
#'-g', |
| 60 |
#'-Og', |
| 61 |
#'-Wno-unused-result', |
| 62 |
#'-Wsign-compare', |
| 63 |
#'-Wall', |
| 64 |
#'-Wextra', |
| 65 |
'-E', |
| 66 |
) |
| 67 |
|
| 68 |
|
| 69 |
def preprocess(filename, |
| 70 |
incldirs=None, |
| 71 |
includes=None, |
| 72 |
macros=None, |
| 73 |
samefiles=None, |
| 74 |
cwd=None, |
| 75 |
): |
| 76 |
if not cwd or not os.path.isabs(cwd): |
| 77 |
cwd = os.path.abspath(cwd or '.') |
| 78 |
filename = _normpath(filename, cwd) |
| 79 |
|
| 80 |
postargs = POST_ARGS |
| 81 |
basename = os.path.basename(filename) |
| 82 |
dirname = os.path.basename(os.path.dirname(filename)) |
| 83 |
if (basename not in FILES_WITHOUT_INTERNAL_CAPI |
| 84 |
and dirname not in DIRS_WITHOUT_INTERNAL_CAPI): |
| 85 |
postargs += ('-DPy_BUILD_CORE=1',) |
| 86 |
|
| 87 |
text = _common.preprocess( |
| 88 |
TOOL, |
| 89 |
filename, |
| 90 |
incldirs=incldirs, |
| 91 |
includes=includes, |
| 92 |
macros=macros, |
| 93 |
#preargs=PRE_ARGS, |
| 94 |
postargs=postargs, |
| 95 |
executable=['gcc'], |
| 96 |
compiler='unix', |
| 97 |
cwd=cwd, |
| 98 |
) |
| 99 |
return _iter_lines(text, filename, samefiles, cwd) |
| 100 |
|
| 101 |
|
| 102 |
def _iter_lines(text, reqfile, samefiles, cwd, raw=False): |
| 103 |
lines = iter(text.splitlines()) |
| 104 |
|
| 105 |
# The first line is special. |
| 106 |
# The next two lines are consistent. |
| 107 |
firstlines = [ |
| 108 |
f'# 0 "{reqfile}"', |
| 109 |
'# 0 "<built-in>"', |
| 110 |
'# 0 "<command-line>"', |
| 111 |
] |
| 112 |
if text.startswith('# 1 '): |
| 113 |
# Some preprocessors emit a lineno of 1 for line-less entries. |
| 114 |
firstlines = [l.replace('# 0 ', '# 1 ') for l in firstlines] |
| 115 |
for expected in firstlines: |
| 116 |
line = next(lines) |
| 117 |
if line != expected: |
| 118 |
raise NotImplementedError((line, expected)) |
| 119 |
|
| 120 |
# Do all the CLI-provided includes. |
| 121 |
filter_reqfile = (lambda f: _filter_reqfile(f, reqfile, samefiles)) |
| 122 |
make_info = (lambda lno: _common.FileInfo(reqfile, lno)) |
| 123 |
last = None |
| 124 |
for line in lines: |
| 125 |
assert last != reqfile, (last,) |
| 126 |
lno, included, flags = _parse_marker_line(line, reqfile) |
| 127 |
if not included: |
| 128 |
raise NotImplementedError((line,)) |
| 129 |
if included == reqfile: |
| 130 |
# This will be the last one. |
| 131 |
assert not flags, (line, flags) |
| 132 |
else: |
| 133 |
assert 1 in flags, (line, flags) |
| 134 |
yield from _iter_top_include_lines( |
| 135 |
lines, |
| 136 |
_normpath(included, cwd), |
| 137 |
cwd, |
| 138 |
filter_reqfile, |
| 139 |
make_info, |
| 140 |
raw, |
| 141 |
) |
| 142 |
last = included |
| 143 |
# The last one is always the requested file. |
| 144 |
assert included == reqfile, (line,) |
| 145 |
|
| 146 |
|
| 147 |
def _iter_top_include_lines(lines, topfile, cwd, |
| 148 |
filter_reqfile, make_info, |
| 149 |
raw): |
| 150 |
partial = 0 # depth |
| 151 |
files = [topfile] |
| 152 |
# We start at 1 in case there are source lines (including blank ones) |
| 153 |
# before the first marker line. Also, we already verified in |
| 154 |
# _parse_marker_line() that the preprocessor reported lno as 1. |
| 155 |
lno = 1 |
| 156 |
for line in lines: |
| 157 |
if line == '# 0 "<command-line>" 2' or line == '# 1 "<command-line>" 2': |
| 158 |
# We're done with this top-level include. |
| 159 |
return |
| 160 |
|
| 161 |
_lno, included, flags = _parse_marker_line(line) |
| 162 |
if included: |
| 163 |
lno = _lno |
| 164 |
included = _normpath(included, cwd) |
| 165 |
# We hit a marker line. |
| 166 |
if 1 in flags: |
| 167 |
# We're entering a file. |
| 168 |
# XXX Cycles are unexpected? |
| 169 |
#assert included not in files, (line, files) |
| 170 |
files.append(included) |
| 171 |
elif 2 in flags: |
| 172 |
# We're returning to a file. |
| 173 |
assert files and included in files, (line, files) |
| 174 |
assert included != files[-1], (line, files) |
| 175 |
while files[-1] != included: |
| 176 |
files.pop() |
| 177 |
# XXX How can a file return to line 1? |
| 178 |
#assert lno > 1, (line, lno) |
| 179 |
else: |
| 180 |
if included == files[-1]: |
| 181 |
# It's the next line from the file. |
| 182 |
assert lno > 1, (line, lno) |
| 183 |
else: |
| 184 |
# We ran into a user-added #LINE directive, |
| 185 |
# which we promptly ignore. |
| 186 |
pass |
| 187 |
elif not files: |
| 188 |
raise NotImplementedError((line,)) |
| 189 |
elif filter_reqfile(files[-1]): |
| 190 |
assert lno is not None, (line, files[-1]) |
| 191 |
if (m := PREPROC_DIRECTIVE_RE.match(line)): |
| 192 |
name, = m.groups() |
| 193 |
if name != 'pragma': |
| 194 |
raise Exception(line) |
| 195 |
else: |
| 196 |
line = re.sub(r'__inline__', 'inline', line) |
| 197 |
if not raw: |
| 198 |
line, partial = _strip_directives(line, partial=partial) |
| 199 |
yield _common.SourceLine( |
| 200 |
make_info(lno), |
| 201 |
'source', |
| 202 |
line or '', |
| 203 |
None, |
| 204 |
) |
| 205 |
lno += 1 |
| 206 |
|
| 207 |
|
| 208 |
def _parse_marker_line(line, reqfile=None): |
| 209 |
m = LINE_MARKER_RE.match(line) |
| 210 |
if not m: |
| 211 |
return None, None, None |
| 212 |
lno, origfile, flags = m.groups() |
| 213 |
lno = int(lno) |
| 214 |
assert origfile not in META_FILES, (line,) |
| 215 |
assert lno > 0, (line, lno) |
| 216 |
flags = set(int(f) for f in flags.split()) if flags else () |
| 217 |
|
| 218 |
if 1 in flags: |
| 219 |
# We're entering a file. |
| 220 |
assert lno == 1, (line, lno) |
| 221 |
assert 2 not in flags, (line,) |
| 222 |
elif 2 in flags: |
| 223 |
# We're returning to a file. |
| 224 |
#assert lno > 1, (line, lno) |
| 225 |
pass |
| 226 |
elif reqfile and origfile == reqfile: |
| 227 |
# We're starting the requested file. |
| 228 |
assert lno == 1, (line, lno) |
| 229 |
assert not flags, (line, flags) |
| 230 |
else: |
| 231 |
# It's the next line from the file. |
| 232 |
assert lno > 1, (line, lno) |
| 233 |
return lno, origfile, flags |
| 234 |
|
| 235 |
|
| 236 |
def _strip_directives(line, partial=0): |
| 237 |
# We assume there are no string literals with parens in directive bodies. |
| 238 |
while partial > 0: |
| 239 |
if not (m := re.match(r'[^{}]*([()])', line)): |
| 240 |
return None, partial |
| 241 |
delim, = m.groups() |
| 242 |
partial += 1 if delim == '(' else -1 # opened/closed |
| 243 |
line = line[m.end():] |
| 244 |
|
| 245 |
line = re.sub(r'__extension__', '', line) |
| 246 |
line = re.sub(r'__thread\b', '_Thread_local', line) |
| 247 |
|
| 248 |
while (m := COMPILER_DIRECTIVE_RE.match(line)): |
| 249 |
before, _, _, closed = m.groups() |
| 250 |
if closed: |
| 251 |
line = f'{before} {line[m.end():]}' |
| 252 |
else: |
| 253 |
after, partial = _strip_directives(line[m.end():], 2) |
| 254 |
line = f'{before} {after or ""}' |
| 255 |
if partial: |
| 256 |
break |
| 257 |
|
| 258 |
return line, partial |
| 259 |
|
| 260 |
|
| 261 |
def _filter_reqfile(current, reqfile, samefiles): |
| 262 |
if current == reqfile: |
| 263 |
return True |
| 264 |
if current == '<stdin>': |
| 265 |
return True |
| 266 |
if current in samefiles: |
| 267 |
return True |
| 268 |
return False |
| 269 |
|
| 270 |
|
| 271 |
def _normpath(filename, cwd): |
| 272 |
assert cwd |
| 273 |
return os.path.normpath(os.path.join(cwd, filename)) |