| 1 |
import contextlib |
| 2 |
import errno |
| 3 |
import os.path |
| 4 |
import socket |
| 5 |
import sys |
| 6 |
import subprocess |
| 7 |
import tempfile |
| 8 |
import unittest |
| 9 |
|
| 10 |
from .. import support |
| 11 |
|
| 12 |
HOST = "localhost" |
| 13 |
HOSTv4 = "127.0.0.1" |
| 14 |
HOSTv6 = "::1" |
| 15 |
|
| 16 |
# WASI SDK 15.0 does not provide gethostname, stub raises OSError ENOTSUP. |
| 17 |
has_gethostname = not support.is_wasi |
| 18 |
|
| 19 |
|
| 20 |
def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): |
| 21 |
"""Returns an unused port that should be suitable for binding. This is |
| 22 |
achieved by creating a temporary socket with the same family and type as |
| 23 |
the 'sock' parameter (default is AF_INET, SOCK_STREAM), and binding it to |
| 24 |
the specified host address (defaults to 0.0.0.0) with the port set to 0, |
| 25 |
eliciting an unused ephemeral port from the OS. The temporary socket is |
| 26 |
then closed and deleted, and the ephemeral port is returned. |
| 27 |
|
| 28 |
Either this method or bind_port() should be used for any tests where a |
| 29 |
server socket needs to be bound to a particular port for the duration of |
| 30 |
the test. Which one to use depends on whether the calling code is creating |
| 31 |
a python socket, or if an unused port needs to be provided in a constructor |
| 32 |
or passed to an external program (i.e. the -accept argument to openssl's |
| 33 |
s_server mode). Always prefer bind_port() over find_unused_port() where |
| 34 |
possible. Hard coded ports should *NEVER* be used. As soon as a server |
| 35 |
socket is bound to a hard coded port, the ability to run multiple instances |
| 36 |
of the test simultaneously on the same host is compromised, which makes the |
| 37 |
test a ticking time bomb in a buildbot environment. On Unix buildbots, this |
| 38 |
may simply manifest as a failed test, which can be recovered from without |
| 39 |
intervention in most cases, but on Windows, the entire python process can |
| 40 |
completely and utterly wedge, requiring someone to log in to the buildbot |
| 41 |
and manually kill the affected process. |
| 42 |
|
| 43 |
(This is easy to reproduce on Windows, unfortunately, and can be traced to |
| 44 |
the SO_REUSEADDR socket option having different semantics on Windows versus |
| 45 |
Unix/Linux. On Unix, you can't have two AF_INET SOCK_STREAM sockets bind, |
| 46 |
listen and then accept connections on identical host/ports. An EADDRINUSE |
| 47 |
OSError will be raised at some point (depending on the platform and |
| 48 |
the order bind and listen were called on each socket). |
| 49 |
|
| 50 |
However, on Windows, if SO_REUSEADDR is set on the sockets, no EADDRINUSE |
| 51 |
will ever be raised when attempting to bind two identical host/ports. When |
| 52 |
accept() is called on each socket, the second caller's process will steal |
| 53 |
the port from the first caller, leaving them both in an awkwardly wedged |
| 54 |
state where they'll no longer respond to any signals or graceful kills, and |
| 55 |
must be forcibly killed via OpenProcess()/TerminateProcess(). |
| 56 |
|
| 57 |
The solution on Windows is to use the SO_EXCLUSIVEADDRUSE socket option |
| 58 |
instead of SO_REUSEADDR, which effectively affords the same semantics as |
| 59 |
SO_REUSEADDR on Unix. Given the propensity of Unix developers in the Open |
| 60 |
Source world compared to Windows ones, this is a common mistake. A quick |
| 61 |
look over OpenSSL's 0.9.8g source shows that they use SO_REUSEADDR when |
| 62 |
openssl.exe is called with the 's_server' option, for example. See |
| 63 |
http://bugs.python.org/issue2550 for more info. The following site also |
| 64 |
has a very thorough description about the implications of both REUSEADDR |
| 65 |
and EXCLUSIVEADDRUSE on Windows: |
| 66 |
https://learn.microsoft.com/windows/win32/winsock/using-so-reuseaddr-and-so-exclusiveaddruse |
| 67 |
|
| 68 |
XXX: although this approach is a vast improvement on previous attempts to |
| 69 |
elicit unused ports, it rests heavily on the assumption that the ephemeral |
| 70 |
port returned to us by the OS won't immediately be dished back out to some |
| 71 |
other process when we close and delete our temporary socket but before our |
| 72 |
calling code has a chance to bind the returned port. We can deal with this |
| 73 |
issue if/when we come across it. |
| 74 |
""" |
| 75 |
|
| 76 |
with socket.socket(family, socktype) as tempsock: |
| 77 |
port = bind_port(tempsock) |
| 78 |
del tempsock |
| 79 |
return port |
| 80 |
|
| 81 |
def bind_port(sock, host=HOST): |
| 82 |
"""Bind the socket to a free port and return the port number. Relies on |
| 83 |
ephemeral ports in order to ensure we are using an unbound port. This is |
| 84 |
important as many tests may be running simultaneously, especially in a |
| 85 |
buildbot environment. This method raises an exception if the sock.family |
| 86 |
is AF_INET and sock.type is SOCK_STREAM, *and* the socket has SO_REUSEADDR |
| 87 |
or SO_REUSEPORT set on it. Tests should *never* set these socket options |
| 88 |
for TCP/IP sockets. The only case for setting these options is testing |
| 89 |
multicasting via multiple UDP sockets. |
| 90 |
|
| 91 |
Additionally, if the SO_EXCLUSIVEADDRUSE socket option is available (i.e. |
| 92 |
on Windows), it will be set on the socket. This will prevent anyone else |
| 93 |
from bind()'ing to our host/port for the duration of the test. |
| 94 |
""" |
| 95 |
|
| 96 |
if sock.family == socket.AF_INET and sock.type == socket.SOCK_STREAM: |
| 97 |
if hasattr(socket, 'SO_REUSEADDR'): |
| 98 |
if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) == 1: |
| 99 |
raise support.TestFailed("tests should never set the " |
| 100 |
"SO_REUSEADDR socket option on " |
| 101 |
"TCP/IP sockets!") |
| 102 |
if hasattr(socket, 'SO_REUSEPORT'): |
| 103 |
try: |
| 104 |
if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 1: |
| 105 |
raise support.TestFailed("tests should never set the " |
| 106 |
"SO_REUSEPORT socket option on " |
| 107 |
"TCP/IP sockets!") |
| 108 |
except OSError: |
| 109 |
# Python's socket module was compiled using modern headers |
| 110 |
# thus defining SO_REUSEPORT but this process is running |
| 111 |
# under an older kernel that does not support SO_REUSEPORT. |
| 112 |
pass |
| 113 |
if hasattr(socket, 'SO_EXCLUSIVEADDRUSE'): |
| 114 |
sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1) |
| 115 |
|
| 116 |
sock.bind((host, 0)) |
| 117 |
port = sock.getsockname()[1] |
| 118 |
return port |
| 119 |
|
| 120 |
def bind_unix_socket(sock, addr): |
| 121 |
"""Bind a unix socket, raising SkipTest if PermissionError is raised.""" |
| 122 |
assert sock.family == socket.AF_UNIX |
| 123 |
try: |
| 124 |
sock.bind(addr) |
| 125 |
except PermissionError: |
| 126 |
sock.close() |
| 127 |
raise unittest.SkipTest('cannot bind AF_UNIX sockets') |
| 128 |
|
| 129 |
def _is_ipv6_enabled(): |
| 130 |
"""Check whether IPv6 is enabled on this host.""" |
| 131 |
if socket.has_ipv6: |
| 132 |
sock = None |
| 133 |
try: |
| 134 |
sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) |
| 135 |
sock.bind((HOSTv6, 0)) |
| 136 |
return True |
| 137 |
except OSError: |
| 138 |
pass |
| 139 |
finally: |
| 140 |
if sock: |
| 141 |
sock.close() |
| 142 |
return False |
| 143 |
|
| 144 |
IPV6_ENABLED = _is_ipv6_enabled() |
| 145 |
|
| 146 |
|
| 147 |
_bind_nix_socket_error = None |
| 148 |
def skip_unless_bind_unix_socket(test): |
| 149 |
"""Decorator for tests requiring a functional bind() for unix sockets.""" |
| 150 |
if not hasattr(socket, 'AF_UNIX'): |
| 151 |
return unittest.skip('No UNIX Sockets')(test) |
| 152 |
global _bind_nix_socket_error |
| 153 |
if _bind_nix_socket_error is None: |
| 154 |
from .os_helper import TESTFN, unlink |
| 155 |
path = TESTFN + "can_bind_unix_socket" |
| 156 |
with socket.socket(socket.AF_UNIX) as sock: |
| 157 |
try: |
| 158 |
sock.bind(path) |
| 159 |
_bind_nix_socket_error = False |
| 160 |
except OSError as e: |
| 161 |
_bind_nix_socket_error = e |
| 162 |
finally: |
| 163 |
unlink(path) |
| 164 |
if _bind_nix_socket_error: |
| 165 |
msg = 'Requires a functional unix bind(): %s' % _bind_nix_socket_error |
| 166 |
return unittest.skip(msg)(test) |
| 167 |
else: |
| 168 |
return test |
| 169 |
|
| 170 |
|
| 171 |
def get_socket_conn_refused_errs(): |
| 172 |
""" |
| 173 |
Get the different socket error numbers ('errno') which can be received |
| 174 |
when a connection is refused. |
| 175 |
""" |
| 176 |
errors = [errno.ECONNREFUSED] |
| 177 |
if hasattr(errno, 'ENETUNREACH'): |
| 178 |
# On Solaris, ENETUNREACH is returned sometimes instead of ECONNREFUSED |
| 179 |
errors.append(errno.ENETUNREACH) |
| 180 |
if hasattr(errno, 'EADDRNOTAVAIL'): |
| 181 |
# bpo-31910: socket.create_connection() fails randomly |
| 182 |
# with EADDRNOTAVAIL on Travis CI |
| 183 |
errors.append(errno.EADDRNOTAVAIL) |
| 184 |
if hasattr(errno, 'EHOSTUNREACH'): |
| 185 |
# bpo-37583: The destination host cannot be reached |
| 186 |
errors.append(errno.EHOSTUNREACH) |
| 187 |
if not IPV6_ENABLED: |
| 188 |
errors.append(errno.EAFNOSUPPORT) |
| 189 |
return errors |
| 190 |
|
| 191 |
|
| 192 |
_NOT_SET = object() |
| 193 |
|
| 194 |
@contextlib.contextmanager |
| 195 |
def transient_internet(resource_name, *, timeout=_NOT_SET, errnos=()): |
| 196 |
"""Return a context manager that raises ResourceDenied when various issues |
| 197 |
with the internet connection manifest themselves as exceptions.""" |
| 198 |
import urllib.error |
| 199 |
if timeout is _NOT_SET: |
| 200 |
timeout = support.INTERNET_TIMEOUT |
| 201 |
|
| 202 |
default_errnos = [ |
| 203 |
('ECONNREFUSED', 111), |
| 204 |
('ECONNRESET', 104), |
| 205 |
('EHOSTUNREACH', 113), |
| 206 |
('ENETUNREACH', 101), |
| 207 |
('ETIMEDOUT', 110), |
| 208 |
# socket.create_connection() fails randomly with |
| 209 |
# EADDRNOTAVAIL on Travis CI. |
| 210 |
('EADDRNOTAVAIL', 99), |
| 211 |
] |
| 212 |
default_gai_errnos = [ |
| 213 |
('EAI_AGAIN', -3), |
| 214 |
('EAI_FAIL', -4), |
| 215 |
('EAI_NONAME', -2), |
| 216 |
('EAI_NODATA', -5), |
| 217 |
# Encountered when trying to resolve IPv6-only hostnames |
| 218 |
('WSANO_DATA', 11004), |
| 219 |
] |
| 220 |
|
| 221 |
denied = support.ResourceDenied("Resource %r is not available" % resource_name) |
| 222 |
captured_errnos = errnos |
| 223 |
gai_errnos = [] |
| 224 |
if not captured_errnos: |
| 225 |
captured_errnos = [getattr(errno, name, num) |
| 226 |
for (name, num) in default_errnos] |
| 227 |
gai_errnos = [getattr(socket, name, num) |
| 228 |
for (name, num) in default_gai_errnos] |
| 229 |
|
| 230 |
def filter_error(err): |
| 231 |
n = getattr(err, 'errno', None) |
| 232 |
if (isinstance(err, TimeoutError) or |
| 233 |
(isinstance(err, socket.gaierror) and n in gai_errnos) or |
| 234 |
(isinstance(err, urllib.error.HTTPError) and |
| 235 |
500 <= err.code <= 599) or |
| 236 |
(isinstance(err, urllib.error.URLError) and |
| 237 |
(("ConnectionRefusedError" in err.reason) or |
| 238 |
("TimeoutError" in err.reason) or |
| 239 |
("EOFError" in err.reason))) or |
| 240 |
n in captured_errnos): |
| 241 |
if not support.verbose: |
| 242 |
sys.stderr.write(denied.args[0] + "\n") |
| 243 |
raise denied from err |
| 244 |
|
| 245 |
old_timeout = socket.getdefaulttimeout() |
| 246 |
try: |
| 247 |
if timeout is not None: |
| 248 |
socket.setdefaulttimeout(timeout) |
| 249 |
yield |
| 250 |
except OSError as err: |
| 251 |
# urllib can wrap original socket errors multiple times (!), we must |
| 252 |
# unwrap to get at the original error. |
| 253 |
while True: |
| 254 |
a = err.args |
| 255 |
if len(a) >= 1 and isinstance(a[0], OSError): |
| 256 |
err = a[0] |
| 257 |
# The error can also be wrapped as args[1]: |
| 258 |
# except socket.error as msg: |
| 259 |
# raise OSError('socket error', msg) from msg |
| 260 |
elif len(a) >= 2 and isinstance(a[1], OSError): |
| 261 |
err = a[1] |
| 262 |
# The error can also be wrapped as __cause__: |
| 263 |
# raise URLError(f"ftp error: {exp}") from exp |
| 264 |
elif isinstance(err, urllib.error.URLError) and err.__cause__: |
| 265 |
err = err.__cause__ |
| 266 |
else: |
| 267 |
break |
| 268 |
filter_error(err) |
| 269 |
raise |
| 270 |
# XXX should we catch generic exceptions and look for their |
| 271 |
# __cause__ or __context__? |
| 272 |
finally: |
| 273 |
socket.setdefaulttimeout(old_timeout) |
| 274 |
|
| 275 |
|
| 276 |
def create_unix_domain_name(): |
| 277 |
""" |
| 278 |
Create a UNIX domain name: socket.bind() argument of a AF_UNIX socket. |
| 279 |
|
| 280 |
Return a path relative to the current directory to get a short path |
| 281 |
(around 27 ASCII characters). |
| 282 |
""" |
| 283 |
return tempfile.mktemp(prefix="test_python_", suffix='.sock', |
| 284 |
dir=os.path.curdir) |
| 285 |
|
| 286 |
|
| 287 |
# consider that sysctl values should not change while tests are running |
| 288 |
_sysctl_cache = {} |
| 289 |
|
| 290 |
def _get_sysctl(name): |
| 291 |
"""Get a sysctl value as an integer.""" |
| 292 |
try: |
| 293 |
return _sysctl_cache[name] |
| 294 |
except KeyError: |
| 295 |
pass |
| 296 |
|
| 297 |
# At least Linux and FreeBSD support the "-n" option |
| 298 |
cmd = ['sysctl', '-n', name] |
| 299 |
proc = subprocess.run(cmd, |
| 300 |
stdout=subprocess.PIPE, |
| 301 |
stderr=subprocess.STDOUT, |
| 302 |
text=True) |
| 303 |
if proc.returncode: |
| 304 |
support.print_warning(f'{' '.join(cmd)!r} command failed with ' |
| 305 |
f'exit code {proc.returncode}') |
| 306 |
# cache the error to only log the warning once |
| 307 |
_sysctl_cache[name] = None |
| 308 |
return None |
| 309 |
output = proc.stdout |
| 310 |
|
| 311 |
# Parse '0\n' to get '0' |
| 312 |
try: |
| 313 |
value = int(output.strip()) |
| 314 |
except Exception as exc: |
| 315 |
support.print_warning(f'Failed to parse {' '.join(cmd)!r} ' |
| 316 |
f'command output {output!r}: {exc!r}') |
| 317 |
# cache the error to only log the warning once |
| 318 |
_sysctl_cache[name] = None |
| 319 |
return None |
| 320 |
|
| 321 |
_sysctl_cache[name] = value |
| 322 |
return value |
| 323 |
|
| 324 |
|
| 325 |
def tcp_blackhole(): |
| 326 |
if not sys.platform.startswith('freebsd'): |
| 327 |
return False |
| 328 |
|
| 329 |
# gh-109015: test if FreeBSD TCP blackhole is enabled |
| 330 |
value = _get_sysctl('net.inet.tcp.blackhole') |
| 331 |
if value is None: |
| 332 |
# don't skip if we fail to get the sysctl value |
| 333 |
return False |
| 334 |
return (value != 0) |
| 335 |
|
| 336 |
|
| 337 |
def skip_if_tcp_blackhole(test): |
| 338 |
"""Decorator skipping test if TCP blackhole is enabled.""" |
| 339 |
skip_if = unittest.skipIf( |
| 340 |
tcp_blackhole(), |
| 341 |
"TCP blackhole is enabled (sysctl net.inet.tcp.blackhole)" |
| 342 |
) |
| 343 |
return skip_if(test) |