Debian Codegraph @

Lib/test/test_asyncio/utils.py

raw
1 """Utilities shared by tests."""
2
3 import asyncio
4 import collections
5 import contextlib
6 import io
7 import logging
8 import os
9 import re
10 import selectors
11 import socket
12 import socketserver
13 import sys
14 import threading
15 import unittest
16 import weakref
17 from ast import literal_eval
18 from unittest import mock
19
20 from http.server import HTTPServer
21 from wsgiref.simple_server import WSGIRequestHandler, WSGIServer
22
23 try:
24 import ssl
25 except ImportError: # pragma: no cover
26 ssl = None
27
28 from asyncio import base_events
29 from asyncio import events
30 from asyncio import format_helpers
31 from asyncio import tasks
32 from asyncio.log import logger
33 from test import support
34 from test.support import socket_helper
35 from test.support import threading_helper
36
37
38 # Use the maximum known clock resolution (gh-75191, gh-110088): Windows
39 # GetTickCount64() has a resolution of 15.6 ms. Use 50 ms to tolerate rounding
40 # issues.
41 CLOCK_RES = 0.050
42
43
44 def data_file(*filename):
45 fullname = os.path.join(support.TEST_HOME_DIR, *filename)
46 if os.path.isfile(fullname):
47 return fullname
48 fullname = os.path.join(os.path.dirname(__file__), '..', *filename)
49 if os.path.isfile(fullname):
50 return fullname
51 raise FileNotFoundError(os.path.join(filename))
52
53
54 ONLYCERT = data_file('certdata', 'ssl_cert.pem')
55 ONLYKEY = data_file('certdata', 'ssl_key.pem')
56 SIGNED_CERTFILE = data_file('certdata', 'keycert3.pem')
57 SIGNING_CA = data_file('certdata', 'pycacert.pem')
58 with open(data_file('certdata', 'keycert3.pem.reference')) as file:
59 PEERCERT = literal_eval(file.read())
60
61 def simple_server_sslcontext():
62 server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
63 server_context.load_cert_chain(ONLYCERT, ONLYKEY)
64 server_context.check_hostname = False
65 server_context.verify_mode = ssl.CERT_NONE
66 return server_context
67
68
69 def simple_client_sslcontext(*, disable_verify=True):
70 client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
71 client_context.check_hostname = False
72 if disable_verify:
73 client_context.verify_mode = ssl.CERT_NONE
74 return client_context
75
76
77 def dummy_ssl_context():
78 if ssl is None:
79 return None
80 else:
81 return simple_client_sslcontext(disable_verify=True)
82
83
84 def run_briefly(loop):
85 async def once():
86 pass
87 gen = once()
88 t = loop.create_task(gen)
89 # Don't log a warning if the task is not done after run_until_complete().
90 # It occurs if the loop is stopped or if a task raises a BaseException.
91 t._log_destroy_pending = False
92 try:
93 loop.run_until_complete(t)
94 finally:
95 gen.close()
96
97
98 def run_until(loop, pred, timeout=support.SHORT_TIMEOUT):
99 delay = 0.001
100 for _ in support.busy_retry(timeout, error=False):
101 if pred():
102 break
103 loop.run_until_complete(tasks.sleep(delay))
104 delay = max(delay * 2, 1.0)
105 else:
106 raise TimeoutError()
107
108
109 def run_once(loop):
110 """Legacy API to run once through the event loop.
111
112 This is the recommended pattern for test code. It will poll the
113 selector once and run all callbacks scheduled in response to I/O
114 events.
115 """
116 loop.call_soon(loop.stop)
117 loop.run_forever()
118
119
120 class SilentWSGIRequestHandler(WSGIRequestHandler):
121
122 def get_stderr(self):
123 return io.StringIO()
124
125 def log_message(self, format, *args):
126 pass
127
128
129 class SilentWSGIServer(WSGIServer):
130
131 request_timeout = support.LOOPBACK_TIMEOUT
132
133 def get_request(self):
134 request, client_addr = super().get_request()
135 request.settimeout(self.request_timeout)
136 return request, client_addr
137
138 def handle_error(self, request, client_address):
139 pass
140
141
142 class SSLWSGIServerMixin:
143
144 def finish_request(self, request, client_address):
145 # The relative location of our test directory (which
146 # contains the ssl key and certificate files) differs
147 # between the stdlib and stand-alone asyncio.
148 # Prefer our own if we can find it.
149 context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
150 context.load_cert_chain(ONLYCERT, ONLYKEY)
151
152 ssock = context.wrap_socket(request, server_side=True)
153 try:
154 self.RequestHandlerClass(ssock, client_address, self)
155 ssock.close()
156 except OSError:
157 # maybe socket has been closed by peer
158 pass
159
160
161 class SSLWSGIServer(SSLWSGIServerMixin, SilentWSGIServer):
162 pass
163
164
165 def _run_test_server(*, address, use_ssl=False, server_cls, server_ssl_cls):
166
167 def loop(environ):
168 size = int(environ['CONTENT_LENGTH'])
169 while size:
170 data = environ['wsgi.input'].read(min(size, 0x10000))
171 yield data
172 size -= len(data)
173
174 def app(environ, start_response):
175 status = '200 OK'
176 headers = [('Content-type', 'text/plain')]
177 start_response(status, headers)
178 if environ['PATH_INFO'] == '/loop':
179 return loop(environ)
180 else:
181 return [b'Test message']
182
183 # Run the test WSGI server in a separate thread in order not to
184 # interfere with event handling in the main thread
185 server_class = server_ssl_cls if use_ssl else server_cls
186 httpd = server_class(address, SilentWSGIRequestHandler)
187 httpd.set_app(app)
188 httpd.address = httpd.server_address
189 server_thread = threading.Thread(
190 target=lambda: httpd.serve_forever(poll_interval=0.05))
191 server_thread.start()
192 try:
193 yield httpd
194 finally:
195 httpd.shutdown()
196 httpd.server_close()
197 server_thread.join()
198
199
200 if hasattr(socket, 'AF_UNIX'):
201
202 class UnixHTTPServer(socketserver.UnixStreamServer, HTTPServer):
203
204 def server_bind(self):
205 socketserver.UnixStreamServer.server_bind(self)
206 self.server_name = '127.0.0.1'
207 self.server_port = 80
208
209
210 class UnixWSGIServer(UnixHTTPServer, WSGIServer):
211
212 request_timeout = support.LOOPBACK_TIMEOUT
213
214 def server_bind(self):
215 UnixHTTPServer.server_bind(self)
216 self.setup_environ()
217
218 def get_request(self):
219 request, client_addr = super().get_request()
220 request.settimeout(self.request_timeout)
221 # Code in the stdlib expects that get_request
222 # will return a socket and a tuple (host, port).
223 # However, this isn't true for UNIX sockets,
224 # as the second return value will be a path;
225 # hence we return some fake data sufficient
226 # to get the tests going
227 return request, ('127.0.0.1', '')
228
229
230 class SilentUnixWSGIServer(UnixWSGIServer):
231
232 def handle_error(self, request, client_address):
233 pass
234
235
236 class UnixSSLWSGIServer(SSLWSGIServerMixin, SilentUnixWSGIServer):
237 pass
238
239
240 def gen_unix_socket_path():
241 return socket_helper.create_unix_domain_name()
242
243
244 @contextlib.contextmanager
245 def unix_socket_path():
246 path = gen_unix_socket_path()
247 try:
248 yield path
249 finally:
250 try:
251 os.unlink(path)
252 except OSError:
253 pass
254
255
256 @contextlib.contextmanager
257 def run_test_unix_server(*, use_ssl=False):
258 with unix_socket_path() as path:
259 yield from _run_test_server(address=path, use_ssl=use_ssl,
260 server_cls=SilentUnixWSGIServer,
261 server_ssl_cls=UnixSSLWSGIServer)
262
263
264 @contextlib.contextmanager
265 def run_test_server(*, host='127.0.0.1', port=0, use_ssl=False):
266 yield from _run_test_server(address=(host, port), use_ssl=use_ssl,
267 server_cls=SilentWSGIServer,
268 server_ssl_cls=SSLWSGIServer)
269
270
271 def echo_datagrams(sock):
272 while True:
273 data, addr = sock.recvfrom(4096)
274 if data == b'STOP':
275 sock.close()
276 break
277 else:
278 sock.sendto(data, addr)
279
280
281 @contextlib.contextmanager
282 def run_udp_echo_server(*, host='127.0.0.1', port=0):
283 addr_info = socket.getaddrinfo(host, port, type=socket.SOCK_DGRAM)
284 family, type, proto, _, sockaddr = addr_info[0]
285 sock = socket.socket(family, type, proto)
286 sock.bind((host, port))
287 sockname = sock.getsockname()
288 thread = threading.Thread(target=lambda: echo_datagrams(sock))
289 thread.start()
290 try:
291 yield sockname
292 finally:
293 # gh-122187: use a separate socket to send the stop message to avoid
294 # TSan reported race on the same socket.
295 sock2 = socket.socket(family, type, proto)
296 sock2.sendto(b'STOP', sockname)
297 sock2.close()
298 thread.join()
299
300
301 def make_test_protocol(base):
302 dct = {}
303 for name in dir(base):
304 if name.startswith('__') and name.endswith('__'):
305 # skip magic names
306 continue
307 dct[name] = MockCallback(return_value=None)
308 return type('TestProtocol', (base,) + base.__bases__, dct)()
309
310
311 class TestSelector(selectors.BaseSelector):
312
313 def __init__(self):
314 self.keys = {}
315
316 def register(self, fileobj, events, data=None):
317 key = selectors.SelectorKey(fileobj, 0, events, data)
318 self.keys[fileobj] = key
319 return key
320
321 def unregister(self, fileobj):
322 return self.keys.pop(fileobj)
323
324 def select(self, timeout):
325 return []
326
327 def get_map(self):
328 return self.keys
329
330
331 class TestLoop(base_events.BaseEventLoop):
332 """Loop for unittests.
333
334 It manages self time directly.
335 If something scheduled to be executed later then
336 on next loop iteration after all ready handlers done
337 generator passed to __init__ is calling.
338
339 Generator should be like this:
340
341 def gen():
342 ...
343 when = yield ...
344 ... = yield time_advance
345
346 Value returned by yield is absolute time of next scheduled handler.
347 Value passed to yield is time advance to move loop's time forward.
348 """
349
350 def __init__(self, gen=None):
351 super().__init__()
352
353 if gen is None:
354 def gen():
355 yield
356 self._check_on_close = False
357 else:
358 self._check_on_close = True
359
360 self._gen = gen()
361 next(self._gen)
362 self._time = 0
363 self._clock_resolution = 1e-9
364 self._timers = []
365 self._selector = TestSelector()
366
367 self.readers = {}
368 self.writers = {}
369 self.reset_counters()
370
371 self._transports = weakref.WeakValueDictionary()
372
373 def time(self):
374 return self._time
375
376 def advance_time(self, advance):
377 """Move test time forward."""
378 if advance:
379 self._time += advance
380
381 def close(self):
382 super().close()
383 if self._check_on_close:
384 try:
385 self._gen.send(0)
386 except StopIteration:
387 pass
388 else: # pragma: no cover
389 raise AssertionError("Time generator is not finished")
390
391 def _add_reader(self, fd, callback, *args):
392 self.readers[fd] = events.Handle(callback, args, self, None)
393
394 def _remove_reader(self, fd):
395 self.remove_reader_count[fd] += 1
396 if fd in self.readers:
397 del self.readers[fd]
398 return True
399 else:
400 return False
401
402 def assert_reader(self, fd, callback, *args):
403 if fd not in self.readers:
404 raise AssertionError(f'fd {fd} is not registered')
405 handle = self.readers[fd]
406 if handle._callback != callback:
407 raise AssertionError(
408 f'unexpected callback: {handle._callback} != {callback}')
409 if handle._args != args:
410 raise AssertionError(
411 f'unexpected callback args: {handle._args} != {args}')
412
413 def assert_no_reader(self, fd):
414 if fd in self.readers:
415 raise AssertionError(f'fd {fd} is registered')
416
417 def _add_writer(self, fd, callback, *args):
418 self.writers[fd] = events.Handle(callback, args, self, None)
419
420 def _remove_writer(self, fd):
421 self.remove_writer_count[fd] += 1
422 if fd in self.writers:
423 del self.writers[fd]
424 return True
425 else:
426 return False
427
428 def assert_writer(self, fd, callback, *args):
429 if fd not in self.writers:
430 raise AssertionError(f'fd {fd} is not registered')
431 handle = self.writers[fd]
432 if handle._callback != callback:
433 raise AssertionError(f'{handle._callback!r} != {callback!r}')
434 if handle._args != args:
435 raise AssertionError(f'{handle._args!r} != {args!r}')
436
437 def _ensure_fd_no_transport(self, fd):
438 if not isinstance(fd, int):
439 try:
440 fd = int(fd.fileno())
441 except (AttributeError, TypeError, ValueError):
442 # This code matches selectors._fileobj_to_fd function.
443 raise ValueError("Invalid file object: "
444 "{!r}".format(fd)) from None
445 try:
446 transport = self._transports[fd]
447 except KeyError:
448 pass
449 else:
450 raise RuntimeError(
451 'File descriptor {!r} is used by transport {!r}'.format(
452 fd, transport))
453
454 def add_reader(self, fd, callback, *args):
455 """Add a reader callback."""
456 self._ensure_fd_no_transport(fd)
457 return self._add_reader(fd, callback, *args)
458
459 def remove_reader(self, fd):
460 """Remove a reader callback."""
461 self._ensure_fd_no_transport(fd)
462 return self._remove_reader(fd)
463
464 def add_writer(self, fd, callback, *args):
465 """Add a writer callback.."""
466 self._ensure_fd_no_transport(fd)
467 return self._add_writer(fd, callback, *args)
468
469 def remove_writer(self, fd):
470 """Remove a writer callback."""
471 self._ensure_fd_no_transport(fd)
472 return self._remove_writer(fd)
473
474 def reset_counters(self):
475 self.remove_reader_count = collections.defaultdict(int)
476 self.remove_writer_count = collections.defaultdict(int)
477
478 def _run_once(self):
479 super()._run_once()
480 for when in self._timers:
481 advance = self._gen.send(when)
482 self.advance_time(advance)
483 self._timers = []
484
485 def call_at(self, when, callback, *args, context=None):
486 self._timers.append(when)
487 return super().call_at(when, callback, *args, context=context)
488
489 def _process_events(self, event_list):
490 return
491
492 def _write_to_self(self):
493 pass
494
495
496 def MockCallback(**kwargs):
497 return mock.Mock(spec=['__call__'], **kwargs)
498
499
500 class MockPattern(str):
501 """A regex based str with a fuzzy __eq__.
502
503 Use this helper with 'mock.assert_called_with', or anywhere
504 where a regex comparison between strings is needed.
505
506 For instance:
507 mock_call.assert_called_with(MockPattern('spam.*ham'))
508 """
509 def __eq__(self, other):
510 return bool(re.search(str(self), other, re.S))
511
512
513 class MockInstanceOf:
514 def __init__(self, type):
515 self._type = type
516
517 def __eq__(self, other):
518 return isinstance(other, self._type)
519
520
521 def get_function_source(func):
522 source = format_helpers._get_function_source(func)
523 if source is None:
524 raise ValueError("unable to get the source of %r" % (func,))
525 return source
526
527
528 class TestCase(unittest.TestCase):
529 @staticmethod
530 def close_loop(loop):
531 if loop._default_executor is not None:
532 if not loop.is_closed():
533 loop.run_until_complete(loop.shutdown_default_executor())
534 else:
535 loop._default_executor.shutdown(wait=True)
536 loop.close()
537
538 def set_event_loop(self, loop, *, cleanup=True):
539 if loop is None:
540 raise AssertionError('loop is None')
541 # ensure that the event loop is passed explicitly in asyncio
542 events.set_event_loop(None)
543 if cleanup:
544 self.addCleanup(self.close_loop, loop)
545
546 def new_test_loop(self, gen=None):
547 loop = TestLoop(gen)
548 self.set_event_loop(loop)
549 return loop
550
551 def setUp(self):
552 self._thread_cleanup = threading_helper.threading_setup()
553
554 def tearDown(self):
555 events.set_event_loop(None)
556
557 # Detect CPython bug #23353: ensure that yield/yield-from is not used
558 # in an except block of a generator
559 self.assertIsNone(sys.exception())
560
561 self.doCleanups()
562 threading_helper.threading_cleanup(*self._thread_cleanup)
563 support.reap_children()
564
565
566 @contextlib.contextmanager
567 def disable_logger():
568 """Context manager to disable asyncio logger.
569
570 For example, it can be used to ignore warnings in debug mode.
571 """
572 old_level = logger.level
573 try:
574 logger.setLevel(logging.CRITICAL+1)
575 yield
576 finally:
577 logger.setLevel(old_level)
578
579
580 def mock_nonblocking_socket(proto=socket.IPPROTO_TCP, type=socket.SOCK_STREAM,
581 family=socket.AF_INET):
582 """Create a mock of a non-blocking socket."""
583 sock = mock.MagicMock(socket.socket)
584 sock.proto = proto
585 sock.type = type
586 sock.family = family
587 sock.gettimeout.return_value = 0.0
588 return sock
589
590
591 async def await_without_task(coro):
592 exc = None
593 def func():
594 try:
595 for _ in coro.__await__():
596 pass
597 except BaseException as err:
598 nonlocal exc
599 exc = err
600 asyncio.get_running_loop().call_soon(func)
601 await asyncio.sleep(0)
602 if exc is not None:
603 raise exc
604
605
606 if sys.platform == 'win32':
607 DefaultEventLoopPolicy = asyncio.windows_events._DefaultEventLoopPolicy
608 else:
609 DefaultEventLoopPolicy = asyncio.unix_events._DefaultEventLoopPolicy