| 1 |
#!/usr/bin/env python3 |
| 2 |
|
| 3 |
""" |
| 4 |
This script should be called *manually* when we want to upgrade SSLError |
| 5 |
`library` and `reason` mnemonics to a more recent OpenSSL version. Note |
| 6 |
that error codes are version specific. |
| 7 |
|
| 8 |
It takes two arguments: |
| 9 |
|
| 10 |
- the path to the OpenSSL folder with the correct git checkout (see below) |
| 11 |
- the path to the header file to be generated, usually |
| 12 |
|
| 13 |
Modules/_ssl_data_<MAJOR><MINOR><PATCH>.h |
| 14 |
|
| 15 |
The OpenSSL git checkout should be at a specific tag, using commands like: |
| 16 |
|
| 17 |
git tag --list 'openssl-*' |
| 18 |
git switch --detach openssl-3.4.1 |
| 19 |
|
| 20 |
After generating the definitions, compare the result with newest pre-existing file. |
| 21 |
You can use a command like: |
| 22 |
|
| 23 |
git diff --no-index Modules/_ssl_data_340.h Modules/_ssl_data_341.h |
| 24 |
|
| 25 |
- If the new version *only* adds new definitions, remove the pre-existing file |
| 26 |
and adjust the #include in _ssl.c to point to the new version. |
| 27 |
- If the new version removes or renumbers some definitions, keep both files and |
| 28 |
add a new #include in _ssl.c. |
| 29 |
|
| 30 |
By convention, the latest OpenSSL mnemonics are gathered in the following file: |
| 31 |
|
| 32 |
Modules/_ssl_data_<MAJOR><MINOR>.h |
| 33 |
|
| 34 |
If those mnemonics are renumbered or removed in a subsequent OpenSSL version, |
| 35 |
the file is renamed to "Modules/_ssl_data_<MAJOR><MINOR><PATCH>.h" and the |
| 36 |
latest mnemonics are stored in the patchless file (see below for an example). |
| 37 |
|
| 38 |
A newly supported OpenSSL version should also be added to: |
| 39 |
|
| 40 |
- Tools/ssl/multissltests.py |
| 41 |
- .github/workflows/build.yml |
| 42 |
|
| 43 |
Example: new mnemonics are added |
| 44 |
-------------------------------- |
| 45 |
Assume that "Modules/_ssl_data_32x.h" contains the latest mnemonics for |
| 46 |
CPython and was generated from OpenSSL 3.2.1. If only new mnemonics are |
| 47 |
added in OpenSSL 3.2.2, the following commands should be executed: |
| 48 |
|
| 49 |
# in the OpenSSL git directory |
| 50 |
git switch --detach openssl-3.2.2 |
| 51 |
|
| 52 |
# in the CPython git directory |
| 53 |
python make_ssl_data.py PATH_TO_OPENSSL_GIT_CLONE Modules/_ssl_data_322.h |
| 54 |
mv Modules/_ssl_data_322.h Modules/_ssl_data_32.h |
| 55 |
|
| 56 |
Example: mnemonics are renamed/removed |
| 57 |
-------------------------------------- |
| 58 |
Assume that the existing file is Modules/_ssl_data_34x.h and is based |
| 59 |
on OpenSSL 3.4.0. Since some mnemonics were renamed in OpenSSL 3.4.1, |
| 60 |
the following commands should be executed: |
| 61 |
|
| 62 |
# in the OpenSSL git directory |
| 63 |
git switch --detach openssl-3.4.1 |
| 64 |
|
| 65 |
# in the CPython git directory |
| 66 |
mv Modules/_ssl_data_34.h Modules/_ssl_data_340.h |
| 67 |
python make_ssl_data.py PATH_TO_OPENSSL_GIT_CLONE Modules/_ssl_data_341.h |
| 68 |
mv Modules/_ssl_data_341.h Modules/_ssl_data_34.h |
| 69 |
""" |
| 70 |
|
| 71 |
import argparse |
| 72 |
import datetime |
| 73 |
import logging |
| 74 |
import operator |
| 75 |
import os |
| 76 |
import re |
| 77 |
import subprocess |
| 78 |
|
| 79 |
|
| 80 |
logger = logging.getLogger(__name__) |
| 81 |
parser = argparse.ArgumentParser( |
| 82 |
formatter_class=argparse.RawTextHelpFormatter, |
| 83 |
description="Generate SSL data headers from OpenSSL sources" |
| 84 |
) |
| 85 |
parser.add_argument("srcdir", help="OpenSSL source directory") |
| 86 |
parser.add_argument( |
| 87 |
"output", nargs="?", default=None, |
| 88 |
help="output file (default: standard output)", |
| 89 |
) |
| 90 |
|
| 91 |
|
| 92 |
def error(format_string, *format_args, **kwargs): |
| 93 |
# do not use parser.error() to avoid printing short help |
| 94 |
logger.error(format_string, *format_args, **kwargs) |
| 95 |
raise SystemExit(1) |
| 96 |
|
| 97 |
|
| 98 |
def _file_search(fname, pat): |
| 99 |
with open(fname, encoding="utf-8") as f: |
| 100 |
for line in f: |
| 101 |
match = pat.search(line) |
| 102 |
if match is not None: |
| 103 |
yield match |
| 104 |
|
| 105 |
|
| 106 |
def parse_err_h(args): |
| 107 |
"""Parse error codes from include/openssl/err.h.in. |
| 108 |
|
| 109 |
Detected lines match (up to spaces) "#define ERR_LIB_<LIBNAME> <ERRCODE>", |
| 110 |
e.g., "# define ERR_LIB_NONE 1". |
| 111 |
""" |
| 112 |
pat = re.compile(r"#\s*define\W+(ERR_LIB_(\w+))\s+(\d+)") |
| 113 |
lib2errnum = {} |
| 114 |
for match in _file_search(args.err_h, pat): |
| 115 |
macroname, libname, num = match.groups() |
| 116 |
if macroname in ['ERR_LIB_OFFSET', 'ERR_LIB_MASK']: |
| 117 |
# ignore: "# define ERR_LIB_OFFSET 23L" |
| 118 |
# ignore: "# define ERR_LIB_MASK 0xFF" |
| 119 |
continue |
| 120 |
actual = int(num) |
| 121 |
expect = lib2errnum.setdefault(libname, actual) |
| 122 |
if actual != expect: |
| 123 |
logger.warning("OpenSSL inconsistency for ERR_LIB_%s (%d != %d)", |
| 124 |
libname, actual, expect) |
| 125 |
return lib2errnum |
| 126 |
|
| 127 |
|
| 128 |
def parse_openssl_error_text(args): |
| 129 |
"""Parse error reasons from crypto/err/openssl.txt. |
| 130 |
|
| 131 |
Detected lines match "<LIBNAME>_R_<ERRNAME>:<ERRCODE>:<MESSAGE>", |
| 132 |
e.g., "ASN1_R_ADDING_OBJECT:171:adding object". The <MESSAGE> part |
| 133 |
is not stored as it will be recovered at runtime when needed. |
| 134 |
""" |
| 135 |
# ignore backslash line continuation (placed before <MESSAGE> if present) |
| 136 |
pat = re.compile(r"^((\w+?)_R_(\w+)):(\d+):") |
| 137 |
seen = {} |
| 138 |
for match in _file_search(args.errtxt, pat): |
| 139 |
reason, libname, errname, num = match.groups() |
| 140 |
if "_F_" in reason: # ignore function codes |
| 141 |
# FEAT(picnixz): in the future, we may want to also check |
| 142 |
# the consistency of the OpenSSL files with an external tool. |
| 143 |
# See https://github.com/python/cpython/issues/132745. |
| 144 |
continue |
| 145 |
yield reason, libname, errname, int(num) |
| 146 |
|
| 147 |
|
| 148 |
def parse_extra_reasons(args): |
| 149 |
"""Parse extra reasons from crypto/err/openssl.ec. |
| 150 |
|
| 151 |
Detected lines are matched against "R <LIBNAME>_R_<ERRNAME> <ERRCODE>", |
| 152 |
e.g., "R SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE 1010". |
| 153 |
""" |
| 154 |
pat = re.compile(r"^R\s+((\w+)_R_(\w+))\s+(\d+)") |
| 155 |
for match in _file_search(args.errcodes, pat): |
| 156 |
reason, libname, errname, num = match.groups() |
| 157 |
yield reason, libname, errname, int(num) |
| 158 |
|
| 159 |
|
| 160 |
def gen_library_codes(args): |
| 161 |
"""Generate table short libname to numeric code.""" |
| 162 |
yield "/* generated from args.lib2errnum */" |
| 163 |
yield "static struct py_ssl_library_code library_codes[] = {" |
| 164 |
for libname in sorted(args.lib2errnum): |
| 165 |
yield f"#ifdef ERR_LIB_{libname}" |
| 166 |
yield f' {{"{libname}", ERR_LIB_{libname}}},' |
| 167 |
yield "#endif" |
| 168 |
yield " {NULL, 0} /* sentinel */" |
| 169 |
yield "};" |
| 170 |
|
| 171 |
|
| 172 |
def gen_error_codes(args): |
| 173 |
"""Generate error code table for error reasons.""" |
| 174 |
yield "/* generated from args.reasons */" |
| 175 |
yield "static struct py_ssl_error_code error_codes[] = {" |
| 176 |
for reason, libname, errname, num in args.reasons: |
| 177 |
yield f" #ifdef {reason}" |
| 178 |
yield f' {{"{errname}", ERR_LIB_{libname}, {reason}}},' |
| 179 |
yield " #else" |
| 180 |
yield f' {{"{errname}", {args.lib2errnum[libname]}, {num}}},' |
| 181 |
yield " #endif" |
| 182 |
yield " {NULL, 0, 0} /* sentinel */" |
| 183 |
yield "};" |
| 184 |
|
| 185 |
|
| 186 |
def get_openssl_git_commit(args): |
| 187 |
git_describe = subprocess.run( |
| 188 |
['git', 'describe', '--long', '--dirty'], |
| 189 |
cwd=args.srcdir, |
| 190 |
capture_output=True, |
| 191 |
encoding='utf-8', |
| 192 |
check=True, |
| 193 |
) |
| 194 |
return git_describe.stdout.strip() |
| 195 |
|
| 196 |
|
| 197 |
def main(args=None): |
| 198 |
args = parser.parse_args(args) |
| 199 |
if not os.path.isdir(args.srcdir): |
| 200 |
error(f"OpenSSL directory not found: {args.srcdir}") |
| 201 |
args.err_h = os.path.join(args.srcdir, "include", "openssl", "err.h") |
| 202 |
if not os.path.isfile(args.err_h): |
| 203 |
# Fall back to infile for OpenSSL 3.0.0 and later. |
| 204 |
args.err_h += ".in" |
| 205 |
args.errcodes = os.path.join(args.srcdir, "crypto", "err", "openssl.ec") |
| 206 |
if not os.path.isfile(args.errcodes): |
| 207 |
error(f"file {args.errcodes} not found in {args.srcdir}") |
| 208 |
args.errtxt = os.path.join(args.srcdir, "crypto", "err", "openssl.txt") |
| 209 |
if not os.path.isfile(args.errtxt): |
| 210 |
error(f"file {args.errtxt} not found in {args.srcdir}") |
| 211 |
|
| 212 |
# [("ERR_LIB_X509", "X509", 11), ...] |
| 213 |
args.lib2errnum = parse_err_h(args) |
| 214 |
|
| 215 |
# [('X509_R_AKID_MISMATCH', 'X509', 'AKID_MISMATCH', 110), ...] |
| 216 |
reasons = [] |
| 217 |
reasons.extend(parse_openssl_error_text(args)) |
| 218 |
reasons.extend(parse_extra_reasons(args)) |
| 219 |
# sort by macro name and numeric error code |
| 220 |
args.reasons = sorted(reasons, key=operator.itemgetter(0, 3)) |
| 221 |
|
| 222 |
commit = get_openssl_git_commit(args) |
| 223 |
lines = [ |
| 224 |
"/* File generated by Tools/ssl/make_ssl_data.py */", |
| 225 |
f"/* Generated on {datetime.datetime.now(datetime.UTC).isoformat()} */", |
| 226 |
f"/* Generated from Git commit {commit} */", |
| 227 |
"", |
| 228 |
] |
| 229 |
lines.extend(gen_library_codes(args)) |
| 230 |
lines.append("") |
| 231 |
lines.extend(gen_error_codes(args)) |
| 232 |
|
| 233 |
if args.output is None: |
| 234 |
for line in lines: |
| 235 |
print(line) |
| 236 |
else: |
| 237 |
with open(args.output, 'w') as output: |
| 238 |
for line in lines: |
| 239 |
print(line, file=output) |
| 240 |
|
| 241 |
|
| 242 |
if __name__ == "__main__": |
| 243 |
main() |