Debian Codegraph @

src/tests/portpicker_test.py

raw
1 #!/usr/bin/python3
2 #
3 # Copyright 2007 Google Inc. All Rights Reserved.
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 # http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 #
17 """Unittests for portpicker."""
18
19 # pylint: disable=invalid-name,protected-access,missing-class-docstring,missing-function-docstring
20
21 from contextlib import ExitStack
22 import errno
23 import os
24 import socket
25 import subprocess
26 import sys
27 import time
28 import unittest
29 from unittest import mock
30
31 import portpicker
32 _winapi = portpicker._winapi
33
34 # pylint: disable=invalid-name,protected-access,missing-class-docstring,missing-function-docstring
35
36
37 class CommonTestMixin:
38 def IsUnusedTCPPort(self, port):
39 return self._bind(port, socket.SOCK_STREAM, socket.IPPROTO_TCP)
40
41 def IsUnusedUDPPort(self, port):
42 return self._bind(port, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
43
44 def setUp(self):
45 super().setUp()
46 # So we can Bind even if portpicker.bind is stubbed out.
47 self._bind = portpicker.bind
48 portpicker._owned_ports.clear()
49 portpicker._free_ports.clear()
50 portpicker._random_ports.clear()
51
52
53 @unittest.skipIf(
54 ('PORTSERVER_ADDRESS' not in os.environ) and
55 not hasattr(socket, 'AF_UNIX'),
56 'no existing port server; test launching code requires AF_UNIX.')
57 class PickUnusedPortTestWithAPortServer(CommonTestMixin, unittest.TestCase):
58
59 @classmethod
60 def setUpClass(cls):
61 cls.portserver_process = None
62 if 'PORTSERVER_ADDRESS' not in os.environ:
63 # Launch a portserver child process for our tests to use if we are
64 # able to. Obviously not host-exclusive, but good for integration
65 # testing purposes on CI without a portserver of its own.
66 cls.portserver_address = '@pid%d-test-ports' % os.getpid()
67 try:
68 cls.portserver_process = subprocess.Popen(
69 ['portserver.py', # Installed in PATH within the venv.
70 '--portserver_address=%s' % cls.portserver_address])
71 except EnvironmentError as err:
72 raise unittest.SkipTest(
73 'Unable to launch portserver.py: %s' % err)
74 linux_addr = '\0' + cls.portserver_address[1:] # The @ means 0.
75 # loop for a few seconds waiting for that socket to work.
76 err = '???'
77 for _ in range(123):
78 time.sleep(0.05)
79 try:
80 ps_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
81 ps_sock.connect(linux_addr)
82 except socket.error as err: # pylint: disable=unused-variable
83 continue
84 ps_sock.close()
85 break
86 else:
87 # The socket failed or never accepted connections, assume our
88 # portserver setup attempt failed and bail out.
89 if cls.portserver_process.poll() is not None:
90 cls.portserver_process.kill()
91 cls.portserver_process.wait()
92 cls.portserver_process = None
93 raise unittest.SkipTest(
94 'Unable to connect to our own portserver.py: %s' % err)
95 # Point child processes at our shiny portserver process.
96 os.environ['PORTSERVER_ADDRESS'] = cls.portserver_address
97
98 @classmethod
99 def tearDownClass(cls):
100 if cls.portserver_process:
101 if os.environ.get('PORTSERVER_ADDRESS') == cls.portserver_address:
102 del os.environ['PORTSERVER_ADDRESS']
103 if cls.portserver_process.poll() is None:
104 cls.portserver_process.kill()
105 cls.portserver_process.wait()
106 cls.portserver_process = None
107
108 def testPickUnusedCanSuccessfullyUsePortServer(self):
109
110 with mock.patch.object(portpicker, '_pick_unused_port_without_server'):
111 portpicker._pick_unused_port_without_server.side_effect = (
112 Exception('eek!')
113 )
114
115 # Since _PickUnusedPortWithoutServer() raises an exception, if we
116 # can successfully obtain a port, the portserver must be working.
117 port = portpicker.pick_unused_port()
118 self.assertTrue(self.IsUnusedTCPPort(port))
119 self.assertTrue(self.IsUnusedUDPPort(port))
120
121 def testPickUnusedCanSuccessfullyUsePortServerAddressKwarg(self):
122
123 with mock.patch.object(portpicker, '_pick_unused_port_without_server'):
124 portpicker._pick_unused_port_without_server.side_effect = (
125 Exception('eek!')
126 )
127
128 # Since _PickUnusedPortWithoutServer() raises an exception, and
129 # we've temporarily removed PORTSERVER_ADDRESS from os.environ, if
130 # we can successfully obtain a port, the portserver must be working.
131 addr = os.environ.pop('PORTSERVER_ADDRESS')
132 try:
133 port = portpicker.pick_unused_port(portserver_address=addr)
134 self.assertTrue(self.IsUnusedTCPPort(port))
135 self.assertTrue(self.IsUnusedUDPPort(port))
136 finally:
137 os.environ['PORTSERVER_ADDRESS'] = addr
138
139 def testGetPortFromPortServer(self):
140 """Exercise the get_port_from_port_server() helper function."""
141 for _ in range(10):
142 port = portpicker.get_port_from_port_server(
143 os.environ['PORTSERVER_ADDRESS'])
144 self.assertTrue(self.IsUnusedTCPPort(port))
145 self.assertTrue(self.IsUnusedUDPPort(port))
146
147
148 class PickUnusedPortTest(CommonTestMixin, unittest.TestCase):
149
150 def testPickUnusedPortActuallyWorks(self):
151 """This test can be flaky."""
152 for _ in range(10):
153 port = portpicker.pick_unused_port()
154 self.assertTrue(self.IsUnusedTCPPort(port))
155 self.assertTrue(self.IsUnusedUDPPort(port))
156
157 def testSendsPidToPortServer(self):
158 with ExitStack() as stack:
159 if _winapi:
160 create_file_mock = mock.Mock()
161 create_file_mock.return_value = 0
162 read_file_mock = mock.Mock()
163 write_file_mock = mock.Mock()
164 read_file_mock.return_value = (b'42768\n', 0)
165 stack.enter_context(
166 mock.patch('_winapi.CreateFile', new=create_file_mock))
167 stack.enter_context(
168 mock.patch('_winapi.WriteFile', new=write_file_mock))
169 stack.enter_context(
170 mock.patch('_winapi.ReadFile', new=read_file_mock))
171 port = portpicker.get_port_from_port_server(
172 'portserver', pid=1234)
173 write_file_mock.assert_called_once_with(0, b'1234\n')
174 else:
175 server = mock.Mock()
176 server.recv.return_value = b'42768\n'
177 stack.enter_context(
178 mock.patch.object(socket, 'socket', return_value=server))
179 port = portpicker.get_port_from_port_server(
180 'portserver', pid=1234)
181 server.sendall.assert_called_once_with(b'1234\n')
182
183 self.assertEqual(port, 42768)
184
185 def testPidDefaultsToOwnPid(self):
186 with ExitStack() as stack:
187 stack.enter_context(
188 mock.patch.object(os, 'getpid', return_value=9876))
189
190 if _winapi:
191 create_file_mock = mock.Mock()
192 create_file_mock.return_value = 0
193 read_file_mock = mock.Mock()
194 write_file_mock = mock.Mock()
195 read_file_mock.return_value = (b'52768\n', 0)
196 stack.enter_context(
197 mock.patch('_winapi.CreateFile', new=create_file_mock))
198 stack.enter_context(
199 mock.patch('_winapi.WriteFile', new=write_file_mock))
200 stack.enter_context(
201 mock.patch('_winapi.ReadFile', new=read_file_mock))
202 port = portpicker.get_port_from_port_server('portserver')
203 write_file_mock.assert_called_once_with(0, b'9876\n')
204 else:
205 server = mock.Mock()
206 server.recv.return_value = b'52768\n'
207 stack.enter_context(
208 mock.patch.object(socket, 'socket', return_value=server))
209 port = portpicker.get_port_from_port_server('portserver')
210 server.sendall.assert_called_once_with(b'9876\n')
211
212 self.assertEqual(port, 52768)
213
214 @mock.patch.dict(os.environ,{'PORTSERVER_ADDRESS': 'portserver'})
215 def testReusesPortServerPorts(self):
216 with ExitStack() as stack:
217 if _winapi:
218 read_file_mock = mock.Mock()
219 read_file_mock.side_effect = [
220 (b'12345\n', 0),
221 (b'23456\n', 0),
222 (b'34567\n', 0),
223 ]
224 stack.enter_context(mock.patch('_winapi.CreateFile'))
225 stack.enter_context(mock.patch('_winapi.WriteFile'))
226 stack.enter_context(
227 mock.patch('_winapi.ReadFile', new=read_file_mock))
228 else:
229 server = mock.Mock()
230 server.recv.side_effect = [b'12345\n', b'23456\n', b'34567\n']
231 stack.enter_context(
232 mock.patch.object(socket, 'socket', return_value=server))
233
234 self.assertEqual(portpicker.pick_unused_port(), 12345)
235 self.assertEqual(portpicker.pick_unused_port(), 23456)
236 portpicker.return_port(12345)
237 self.assertEqual(portpicker.pick_unused_port(), 12345)
238
239 @mock.patch.dict(os.environ,{'PORTSERVER_ADDRESS': ''})
240 def testDoesntReuseRandomPorts(self):
241 ports = set()
242 for _ in range(10):
243 try:
244 port = portpicker.pick_unused_port()
245 except portpicker.NoFreePortFoundError:
246 # This sometimes happens when not using portserver. Just
247 # skip to the next attempt.
248 continue
249 ports.add(port)
250 portpicker.return_port(port)
251 self.assertGreater(len(ports), 5) # Allow some random reuse.
252
253 def testReturnsReservedPorts(self):
254 with mock.patch.object(portpicker, '_pick_unused_port_without_server'):
255 portpicker._pick_unused_port_without_server.side_effect = (
256 Exception('eek!'))
257 # Arbitrary port. In practice you should get this from somewhere
258 # that assigns ports.
259 reserved_port = 28465
260 portpicker.add_reserved_port(reserved_port)
261 ports = set()
262 for _ in range(10):
263 port = portpicker.pick_unused_port()
264 ports.add(port)
265 portpicker.return_port(port)
266 self.assertEqual(len(ports), 1)
267 self.assertEqual(ports.pop(), reserved_port)
268
269 @mock.patch.dict(os.environ,{'PORTSERVER_ADDRESS': ''})
270 def testFallsBackToRandomAfterRunningOutOfReservedPorts(self):
271 # Arbitrary port. In practice you should get this from somewhere
272 # that assigns ports.
273 reserved_port = 23456
274 portpicker.add_reserved_port(reserved_port)
275 self.assertEqual(portpicker.pick_unused_port(), reserved_port)
276 self.assertNotEqual(portpicker.pick_unused_port(), reserved_port)
277
278 def testRandomlyChosenPorts(self):
279 # Unless this box is under an overwhelming socket load, this test
280 # will heavily exercise the "pick a port randomly" part of the
281 # port picking code, but may never hit the "OS assigns a port"
282 # code.
283 ports = 0
284 for _ in range(100):
285 try:
286 port = portpicker._pick_unused_port_without_server()
287 except portpicker.NoFreePortFoundError:
288 # Without the portserver, pick_unused_port can sometimes fail
289 # to find a free port. Check that it passes most of the time.
290 continue
291 self.assertTrue(self.IsUnusedTCPPort(port))
292 self.assertTrue(self.IsUnusedUDPPort(port))
293 ports += 1
294 # Getting a port shouldn't have failed very often, even on machines
295 # with a heavy socket load.
296 self.assertGreater(ports, 95)
297
298 def testOSAssignedPorts(self):
299 self.last_assigned_port = None
300
301 def error_for_explicit_ports(port, socket_type, socket_proto):
302 # Only successfully return a port if an OS-assigned port is
303 # requested, or if we're checking that the last OS-assigned port
304 # is unused on the other protocol.
305 if port in (0, self.last_assigned_port):
306 self.last_assigned_port = self._bind(port, socket_type,
307 socket_proto)
308 return self.last_assigned_port
309 return None
310
311 with mock.patch.object(portpicker, 'bind', error_for_explicit_ports):
312 # Without server, this can be little flaky, so check that it
313 # passes most of the time.
314 ports = 0
315 for _ in range(100):
316 try:
317 port = portpicker._pick_unused_port_without_server()
318 except portpicker.NoFreePortFoundError:
319 continue
320 self.assertTrue(self.IsUnusedTCPPort(port))
321 self.assertTrue(self.IsUnusedUDPPort(port))
322 ports += 1
323 self.assertGreater(ports, 70)
324
325 def pickUnusedPortWithoutServer(self):
326 # Try a few times to pick a port, to avoid flakiness and to make sure
327 # the code path we want was exercised.
328 for _ in range(5):
329 try:
330 port = portpicker._pick_unused_port_without_server()
331 except portpicker.NoFreePortFoundError:
332 continue
333 else:
334 self.assertTrue(self.IsUnusedTCPPort(port))
335 self.assertTrue(self.IsUnusedUDPPort(port))
336 return
337 self.fail("Failed to find a free port")
338
339 def testPickPortsWithoutServer(self):
340 # Test the first part of _pick_unused_port_without_server, which
341 # tries a few random ports and checks is_port_free.
342 self.pickUnusedPortWithoutServer()
343
344 # Now test the second part, the fallback from above, which asks the
345 # OS for a port.
346 def mock_port_free(unused_port):
347 return False
348
349 with mock.patch.object(portpicker, 'is_port_free', mock_port_free):
350 self.pickUnusedPortWithoutServer()
351
352 def checkIsPortFree(self):
353 """This might be flaky unless this test is run with a portserver."""
354 # The port should be free initially.
355 port = portpicker.pick_unused_port()
356 self.assertTrue(portpicker.is_port_free(port))
357
358 cases = [
359 (socket.AF_INET, socket.SOCK_STREAM, None),
360 (socket.AF_INET6, socket.SOCK_STREAM, 1),
361 (socket.AF_INET, socket.SOCK_DGRAM, None),
362 (socket.AF_INET6, socket.SOCK_DGRAM, 1),
363 ]
364
365 # Using v6only=0 on Windows doesn't result in collisions
366 if not _winapi:
367 cases.extend([
368 (socket.AF_INET6, socket.SOCK_STREAM, 0),
369 (socket.AF_INET6, socket.SOCK_DGRAM, 0),
370 ])
371
372 for (sock_family, sock_type, v6only) in cases:
373 # Occupy the port on a subset of possible protocols.
374 try:
375 sock = socket.socket(sock_family, sock_type, 0)
376 except socket.error:
377 print('Kernel does not support sock_family=%d' % sock_family,
378 file=sys.stderr)
379 # Skip this case, since we cannot occupy a port.
380 continue
381
382 if not hasattr(socket, 'IPPROTO_IPV6'):
383 v6only = None
384
385 if v6only is not None:
386 try:
387 sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY,
388 v6only)
389 except socket.error:
390 print('Kernel does not support IPV6_V6ONLY=%d' % v6only,
391 file=sys.stderr)
392 # Don't care; just proceed with the default.
393
394 # Socket may have been taken in the mean time, so catch the
395 # socket.error with errno set to EADDRINUSE and skip this
396 # attempt.
397 try:
398 sock.bind(('', port))
399 except socket.error as e:
400 if e.errno == errno.EADDRINUSE:
401 raise portpicker.NoFreePortFoundError
402 raise
403
404 # The port should be busy.
405 self.assertFalse(portpicker.is_port_free(port))
406 sock.close()
407
408 # Now it's free again.
409 self.assertTrue(portpicker.is_port_free(port))
410
411 def testIsPortFree(self):
412 # This can be quite flaky on a busy host, so try a few times.
413 for _ in range(10):
414 try:
415 self.checkIsPortFree()
416 except portpicker.NoFreePortFoundError:
417 pass
418 else:
419 return
420 self.fail("checkPortIsFree failed every time.")
421
422 def testIsPortFreeException(self):
423 port = portpicker.pick_unused_port()
424 with mock.patch.object(socket, 'socket') as mock_sock:
425 mock_sock.side_effect = socket.error('fake socket error', 0)
426 self.assertFalse(portpicker.is_port_free(port))
427
428 def testThatLegacyCapWordsAPIsExist(self):
429 """The original APIs were CapWords style, 1.1 added PEP8 names."""
430 self.assertEqual(portpicker.bind, portpicker.Bind)
431 self.assertEqual(portpicker.is_port_free, portpicker.IsPortFree)
432 self.assertEqual(portpicker.pick_unused_port, portpicker.PickUnusedPort)
433 self.assertEqual(portpicker.get_port_from_port_server,
434 portpicker.GetPortFromPortServer)
435
436
437 def get_open_listen_tcp_ports():
438 netstat = subprocess.run(['netstat', '-lnt'], capture_output=True,
439 encoding='utf-8')
440 if netstat.returncode != 0:
441 raise unittest.SkipTest('Unable to run netstat -lnt to list binds.')
442 rows = (line.split() for line in netstat.stdout.splitlines())
443 listen_addrs = (row[3] for row in rows if row[0].startswith('tcp'))
444 listen_ports = [int(addr.split(':')[-1]) for addr in listen_addrs]
445 return listen_ports
446
447
448 @unittest.skipUnless((sys.executable and os.access(sys.executable, os.X_OK))
449 or (os.environ.get('TEST_PORTPICKER_CLI') and
450 os.access(os.environ['TEST_PORTPICKER_CLI'], os.X_OK)),
451 'sys.executable portpicker.__file__ not launchable and '
452 ' no TEST_PORTPICKER_CLI supplied.')
453 class PortpickerCommandLineTests(unittest.TestCase):
454 def setUp(self):
455 self.main_py = portpicker.__file__
456
457 def _run_portpicker(self, pp_args, env_override=None):
458 env = dict(os.environ)
459 if env_override:
460 env.update(env_override)
461 if os.environ.get('TEST_PORTPICKER_CLI'):
462 pp_command = [os.environ['TEST_PORTPICKER_CLI']]
463 else:
464 pp_command = [sys.executable, '-m', 'portpicker']
465 return subprocess.run(pp_command + pp_args,
466 capture_output=True,
467 env=env,
468 encoding='utf-8',
469 check=False)
470
471 def test_command_line_help(self):
472 cmd = self._run_portpicker(['-h'])
473 self.assertNotEqual(0, cmd.returncode)
474 self.assertIn('usage', cmd.stdout)
475 self.assertIn('passed an arg', cmd.stdout)
476 cmd = self._run_portpicker(['--help'])
477 self.assertNotEqual(0, cmd.returncode)
478 self.assertIn('usage', cmd.stdout)
479 self.assertIn('passed an arg', cmd.stdout)
480
481 def test_command_line_help_text_dedented(self):
482 cmd = self._run_portpicker(['-h'])
483 self.assertNotEqual(0, cmd.returncode)
484 self.assertIn('\nIf passed an arg', cmd.stdout)
485 self.assertIn('\n #!/bin/bash', cmd.stdout)
486 self.assertIn('\nOlder versions ', cmd.stdout)
487
488 def test_command_line_interface(self):
489 cmd = self._run_portpicker([str(os.getpid())])
490 cmd.check_returncode()
491 port = int(cmd.stdout)
492 self.assertNotEqual(0, port, msg=cmd)
493 listen_ports = sorted(get_open_listen_tcp_ports())
494 self.assertNotIn(port, listen_ports, msg='expected nothing to be bound to port.')
495
496 def test_command_line_interface_no_portserver(self):
497 cmd = self._run_portpicker([str(os.getpid())],
498 env_override={'PORTSERVER_ADDRESS': ''})
499 cmd.check_returncode()
500 port = int(cmd.stdout)
501 self.assertNotEqual(0, port, msg=cmd)
502 listen_ports = sorted(get_open_listen_tcp_ports())
503 self.assertNotIn(port, listen_ports, msg='expected nothing to be bound to port.')
504
505 def test_command_line_interface_no_portserver_bind_timeout(self):
506 # This test is timing sensitive and leaves that bind process hanging
507 # around consuming resources until it dies on its own unless the test
508 # runner kills the process group upon exit.
509 timeout = 9.5
510 before = time.monotonic()
511 cmd = self._run_portpicker([str(os.getpid()), str(timeout)],
512 env_override={'PORTSERVER_ADDRESS': ''})
513 self.assertEqual(0, cmd.returncode, msg=(cmd.stdout, cmd.stderr))
514 port = int(cmd.stdout)
515 self.assertNotEqual(0, port, msg=cmd)
516 if 'WARNING' in cmd.stderr:
517 raise unittest.SkipTest('bind timeout not supported on this platform.')
518 listen_ports = sorted(get_open_listen_tcp_ports())
519 self.assertIn(port, listen_ports, msg='expected port to be bound. '
520 '%f seconds elapsed of %f bind timeout.' %
521 (time.monotonic() - before, timeout))
522
523
524 if __name__ == '__main__':
525 unittest.main()