Debian Codegraph @

Tools/ssl/make_ssl_data.py

raw
1 #!/usr/bin/env python3
2
3 """
4 This script should be called *manually* when we want to upgrade SSLError
5 `library` and `reason` mnemonics to a more recent OpenSSL version. Note
6 that error codes are version specific.
7
8 It takes two arguments:
9
10 - the path to the OpenSSL folder with the correct git checkout (see below)
11 - the path to the header file to be generated, usually
12
13 Modules/_ssl_data_<MAJOR><MINOR><PATCH>.h
14
15 The OpenSSL git checkout should be at a specific tag, using commands like:
16
17 git tag --list 'openssl-*'
18 git switch --detach openssl-3.4.1
19
20 After generating the definitions, compare the result with newest pre-existing file.
21 You can use a command like:
22
23 git diff --no-index Modules/_ssl_data_340.h Modules/_ssl_data_341.h
24
25 - If the new version *only* adds new definitions, remove the pre-existing file
26 and adjust the #include in _ssl.c to point to the new version.
27 - If the new version removes or renumbers some definitions, keep both files and
28 add a new #include in _ssl.c.
29
30 By convention, the latest OpenSSL mnemonics are gathered in the following file:
31
32 Modules/_ssl_data_<MAJOR><MINOR>.h
33
34 If those mnemonics are renumbered or removed in a subsequent OpenSSL version,
35 the file is renamed to "Modules/_ssl_data_<MAJOR><MINOR><PATCH>.h" and the
36 latest mnemonics are stored in the patchless file (see below for an example).
37
38 A newly supported OpenSSL version should also be added to:
39
40 - Tools/ssl/multissltests.py
41 - .github/workflows/build.yml
42
43 Example: new mnemonics are added
44 --------------------------------
45 Assume that "Modules/_ssl_data_32x.h" contains the latest mnemonics for
46 CPython and was generated from OpenSSL 3.2.1. If only new mnemonics are
47 added in OpenSSL 3.2.2, the following commands should be executed:
48
49 # in the OpenSSL git directory
50 git switch --detach openssl-3.2.2
51
52 # in the CPython git directory
53 python make_ssl_data.py PATH_TO_OPENSSL_GIT_CLONE Modules/_ssl_data_322.h
54 mv Modules/_ssl_data_322.h Modules/_ssl_data_32.h
55
56 Example: mnemonics are renamed/removed
57 --------------------------------------
58 Assume that the existing file is Modules/_ssl_data_34x.h and is based
59 on OpenSSL 3.4.0. Since some mnemonics were renamed in OpenSSL 3.4.1,
60 the following commands should be executed:
61
62 # in the OpenSSL git directory
63 git switch --detach openssl-3.4.1
64
65 # in the CPython git directory
66 mv Modules/_ssl_data_34.h Modules/_ssl_data_340.h
67 python make_ssl_data.py PATH_TO_OPENSSL_GIT_CLONE Modules/_ssl_data_341.h
68 mv Modules/_ssl_data_341.h Modules/_ssl_data_34.h
69 """
70
71 import argparse
72 import datetime
73 import logging
74 import operator
75 import os
76 import re
77 import subprocess
78
79
80 logger = logging.getLogger(__name__)
81 parser = argparse.ArgumentParser(
82 formatter_class=argparse.RawTextHelpFormatter,
83 description="Generate SSL data headers from OpenSSL sources"
84 )
85 parser.add_argument("srcdir", help="OpenSSL source directory")
86 parser.add_argument(
87 "output", nargs="?", default=None,
88 help="output file (default: standard output)",
89 )
90
91
92 def error(format_string, *format_args, **kwargs):
93 # do not use parser.error() to avoid printing short help
94 logger.error(format_string, *format_args, **kwargs)
95 raise SystemExit(1)
96
97
98 def _file_search(fname, pat):
99 with open(fname, encoding="utf-8") as f:
100 for line in f:
101 match = pat.search(line)
102 if match is not None:
103 yield match
104
105
106 def parse_err_h(args):
107 """Parse error codes from include/openssl/err.h.in.
108
109 Detected lines match (up to spaces) "#define ERR_LIB_<LIBNAME> <ERRCODE>",
110 e.g., "# define ERR_LIB_NONE 1".
111 """
112 pat = re.compile(r"#\s*define\W+(ERR_LIB_(\w+))\s+(\d+)")
113 lib2errnum = {}
114 for match in _file_search(args.err_h, pat):
115 macroname, libname, num = match.groups()
116 if macroname in ['ERR_LIB_OFFSET', 'ERR_LIB_MASK']:
117 # ignore: "# define ERR_LIB_OFFSET 23L"
118 # ignore: "# define ERR_LIB_MASK 0xFF"
119 continue
120 actual = int(num)
121 expect = lib2errnum.setdefault(libname, actual)
122 if actual != expect:
123 logger.warning("OpenSSL inconsistency for ERR_LIB_%s (%d != %d)",
124 libname, actual, expect)
125 return lib2errnum
126
127
128 def parse_openssl_error_text(args):
129 """Parse error reasons from crypto/err/openssl.txt.
130
131 Detected lines match "<LIBNAME>_R_<ERRNAME>:<ERRCODE>:<MESSAGE>",
132 e.g., "ASN1_R_ADDING_OBJECT:171:adding object". The <MESSAGE> part
133 is not stored as it will be recovered at runtime when needed.
134 """
135 # ignore backslash line continuation (placed before <MESSAGE> if present)
136 pat = re.compile(r"^((\w+?)_R_(\w+)):(\d+):")
137 seen = {}
138 for match in _file_search(args.errtxt, pat):
139 reason, libname, errname, num = match.groups()
140 if "_F_" in reason: # ignore function codes
141 # FEAT(picnixz): in the future, we may want to also check
142 # the consistency of the OpenSSL files with an external tool.
143 # See https://github.com/python/cpython/issues/132745.
144 continue
145 yield reason, libname, errname, int(num)
146
147
148 def parse_extra_reasons(args):
149 """Parse extra reasons from crypto/err/openssl.ec.
150
151 Detected lines are matched against "R <LIBNAME>_R_<ERRNAME> <ERRCODE>",
152 e.g., "R SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE 1010".
153 """
154 pat = re.compile(r"^R\s+((\w+)_R_(\w+))\s+(\d+)")
155 for match in _file_search(args.errcodes, pat):
156 reason, libname, errname, num = match.groups()
157 yield reason, libname, errname, int(num)
158
159
160 def gen_library_codes(args):
161 """Generate table short libname to numeric code."""
162 yield "/* generated from args.lib2errnum */"
163 yield "static struct py_ssl_library_code library_codes[] = {"
164 for libname in sorted(args.lib2errnum):
165 yield f"#ifdef ERR_LIB_{libname}"
166 yield f' {{"{libname}", ERR_LIB_{libname}}},'
167 yield "#endif"
168 yield " {NULL, 0} /* sentinel */"
169 yield "};"
170
171
172 def gen_error_codes(args):
173 """Generate error code table for error reasons."""
174 yield "/* generated from args.reasons */"
175 yield "static struct py_ssl_error_code error_codes[] = {"
176 for reason, libname, errname, num in args.reasons:
177 yield f" #ifdef {reason}"
178 yield f' {{"{errname}", ERR_LIB_{libname}, {reason}}},'
179 yield " #else"
180 yield f' {{"{errname}", {args.lib2errnum[libname]}, {num}}},'
181 yield " #endif"
182 yield " {NULL, 0, 0} /* sentinel */"
183 yield "};"
184
185
186 def get_openssl_git_commit(args):
187 git_describe = subprocess.run(
188 ['git', 'describe', '--long', '--dirty'],
189 cwd=args.srcdir,
190 capture_output=True,
191 encoding='utf-8',
192 check=True,
193 )
194 return git_describe.stdout.strip()
195
196
197 def main(args=None):
198 args = parser.parse_args(args)
199 if not os.path.isdir(args.srcdir):
200 error(f"OpenSSL directory not found: {args.srcdir}")
201 args.err_h = os.path.join(args.srcdir, "include", "openssl", "err.h")
202 if not os.path.isfile(args.err_h):
203 # Fall back to infile for OpenSSL 3.0.0 and later.
204 args.err_h += ".in"
205 args.errcodes = os.path.join(args.srcdir, "crypto", "err", "openssl.ec")
206 if not os.path.isfile(args.errcodes):
207 error(f"file {args.errcodes} not found in {args.srcdir}")
208 args.errtxt = os.path.join(args.srcdir, "crypto", "err", "openssl.txt")
209 if not os.path.isfile(args.errtxt):
210 error(f"file {args.errtxt} not found in {args.srcdir}")
211
212 # [("ERR_LIB_X509", "X509", 11), ...]
213 args.lib2errnum = parse_err_h(args)
214
215 # [('X509_R_AKID_MISMATCH', 'X509', 'AKID_MISMATCH', 110), ...]
216 reasons = []
217 reasons.extend(parse_openssl_error_text(args))
218 reasons.extend(parse_extra_reasons(args))
219 # sort by macro name and numeric error code
220 args.reasons = sorted(reasons, key=operator.itemgetter(0, 3))
221
222 commit = get_openssl_git_commit(args)
223 lines = [
224 "/* File generated by Tools/ssl/make_ssl_data.py */",
225 f"/* Generated on {datetime.datetime.now(datetime.UTC).isoformat()} */",
226 f"/* Generated from Git commit {commit} */",
227 "",
228 ]
229 lines.extend(gen_library_codes(args))
230 lines.append("")
231 lines.extend(gen_error_codes(args))
232
233 if args.output is None:
234 for line in lines:
235 print(line)
236 else:
237 with open(args.output, 'w') as output:
238 for line in lines:
239 print(line, file=output)
240
241
242 if __name__ == "__main__":
243 main()