| 1 |
#!/usr/bin/python3 |
| 2 |
# |
| 3 |
# Copyright 2007 Google Inc. All Rights Reserved. |
| 4 |
# |
| 5 |
# Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 |
# you may not use this file except in compliance with the License. |
| 7 |
# You may obtain a copy of the License at |
| 8 |
# |
| 9 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 10 |
# |
| 11 |
# Unless required by applicable law or agreed to in writing, software |
| 12 |
# distributed under the License is distributed on an "AS IS" BASIS, |
| 13 |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 |
# See the License for the specific language governing permissions and |
| 15 |
# limitations under the License. |
| 16 |
# |
| 17 |
"""Pure python code for finding unused ports on a host. |
| 18 |
|
| 19 |
This module provides a pick_unused_port() function. |
| 20 |
It can also be called via the command line for use in shell scripts. |
| 21 |
When called from the command line, it takes one optional argument, which, |
| 22 |
if given, is sent to portserver instead of portpicker's PID. |
| 23 |
To reserve a port for the lifetime of a bash script, use $BASHPID as this |
| 24 |
argument. |
| 25 |
|
| 26 |
There is a race condition between picking a port and your application code |
| 27 |
binding to it. The use of a port server to prevent that is recommended on |
| 28 |
loaded test hosts running many tests at a time. |
| 29 |
|
| 30 |
If your code can accept a bound socket as input rather than being handed a |
| 31 |
port number consider using socket.bind(('localhost', 0)) to bind to an |
| 32 |
available port without a race condition rather than using this library. |
| 33 |
|
| 34 |
Typical usage: |
| 35 |
test_port = portpicker.pick_unused_port() |
| 36 |
""" |
| 37 |
|
| 38 |
# pylint: disable=consider-using-f-string |
| 39 |
# Some people still use this on old Pythons despite our test matrix and |
| 40 |
# supported versions. Be kind for now, until it gets in our way. |
| 41 |
from __future__ import print_function |
| 42 |
|
| 43 |
import logging |
| 44 |
import os |
| 45 |
import random |
| 46 |
import socket |
| 47 |
import sys |
| 48 |
import time |
| 49 |
|
| 50 |
_winapi = None # pylint: disable=invalid-name |
| 51 |
if sys.platform == 'win32': |
| 52 |
try: |
| 53 |
import _winapi |
| 54 |
except ImportError: |
| 55 |
_winapi = None |
| 56 |
|
| 57 |
# The legacy Bind, IsPortFree, etc. names are not exported. |
| 58 |
__all__ = ('bind', 'is_port_free', 'pick_unused_port', 'return_port', |
| 59 |
'add_reserved_port', 'get_port_from_port_server') |
| 60 |
|
| 61 |
_PROTOS = [(socket.SOCK_STREAM, socket.IPPROTO_TCP), |
| 62 |
(socket.SOCK_DGRAM, socket.IPPROTO_UDP)] |
| 63 |
|
| 64 |
|
| 65 |
# Ports that are currently available to be given out. |
| 66 |
_free_ports = set() |
| 67 |
|
| 68 |
# Ports that are reserved or from the portserver that may be returned. |
| 69 |
_owned_ports = set() |
| 70 |
|
| 71 |
# Ports that we chose randomly that may be returned. |
| 72 |
_random_ports = set() |
| 73 |
|
| 74 |
|
| 75 |
class NoFreePortFoundError(Exception): |
| 76 |
"""Exception indicating that no free port could be found.""" |
| 77 |
|
| 78 |
|
| 79 |
def add_reserved_port(port): |
| 80 |
"""Add a port that was acquired by means other than the port server.""" |
| 81 |
_free_ports.add(port) |
| 82 |
|
| 83 |
|
| 84 |
def return_port(port): |
| 85 |
"""Return a port that is no longer being used so it can be reused.""" |
| 86 |
if port in _random_ports: |
| 87 |
_random_ports.remove(port) |
| 88 |
elif port in _owned_ports: |
| 89 |
_owned_ports.remove(port) |
| 90 |
_free_ports.add(port) |
| 91 |
elif port in _free_ports: |
| 92 |
logging.info("Returning a port that was already returned: %s", port) |
| 93 |
else: |
| 94 |
logging.info("Returning a port that wasn't given by portpicker: %s", |
| 95 |
port) |
| 96 |
|
| 97 |
|
| 98 |
def bind(port, socket_type, socket_proto): |
| 99 |
"""Try to bind to a socket of the specified type, protocol, and port. |
| 100 |
|
| 101 |
This is primarily a helper function for PickUnusedPort, used to see |
| 102 |
if a particular port number is available. |
| 103 |
|
| 104 |
For the port to be considered available, the kernel must support at least |
| 105 |
one of (IPv6, IPv4), and the port must be available on each supported |
| 106 |
family. |
| 107 |
|
| 108 |
Args: |
| 109 |
port: The port number to bind to, or 0 to have the OS pick a free port. |
| 110 |
socket_type: The type of the socket (ex: socket.SOCK_STREAM). |
| 111 |
socket_proto: The protocol of the socket (ex: socket.IPPROTO_TCP). |
| 112 |
|
| 113 |
Returns: |
| 114 |
The port number on success or None on failure. |
| 115 |
""" |
| 116 |
return _bind(port, socket_type, socket_proto) |
| 117 |
|
| 118 |
|
| 119 |
def _bind(port, socket_type, socket_proto, return_socket=None, |
| 120 |
return_family=socket.AF_INET6): |
| 121 |
"""Internal implementation of bind. |
| 122 |
|
| 123 |
Args: |
| 124 |
port, socket_type, socket_proto: see bind(). |
| 125 |
return_socket: If supplied, a list that we will append an open bound |
| 126 |
reuseaddr socket on the port in question to. |
| 127 |
return_family: The socket family to return in return_socket. |
| 128 |
|
| 129 |
Returns: |
| 130 |
The port number on success or None on failure. |
| 131 |
""" |
| 132 |
# Our return family must come last when returning a bound socket |
| 133 |
# as we cannot keep it bound while testing a bind on the other |
| 134 |
# family with many network stack configurations. |
| 135 |
if return_socket is None or return_family == socket.AF_INET: |
| 136 |
socket_families = (socket.AF_INET6, socket.AF_INET) |
| 137 |
elif return_family == socket.AF_INET6: |
| 138 |
socket_families = (socket.AF_INET, socket.AF_INET6) |
| 139 |
else: |
| 140 |
raise ValueError('unknown return_family %s' % return_family) |
| 141 |
got_socket = False |
| 142 |
for family in socket_families: |
| 143 |
try: |
| 144 |
sock = socket.socket(family, socket_type, socket_proto) |
| 145 |
got_socket = True |
| 146 |
except socket.error: |
| 147 |
continue |
| 148 |
try: |
| 149 |
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) |
| 150 |
sock.bind(('', port)) |
| 151 |
if socket_type == socket.SOCK_STREAM: |
| 152 |
sock.listen(1) |
| 153 |
port = sock.getsockname()[1] |
| 154 |
except socket.error: |
| 155 |
return None |
| 156 |
finally: |
| 157 |
if return_socket is None or family != return_family: |
| 158 |
try: |
| 159 |
# Adding this resolved 1 in ~500 flakiness that we were |
| 160 |
# seeing from an integration test framework managing a set |
| 161 |
# of ports with is_port_free(). close() doesn't move the |
| 162 |
# TCP state machine along quickly. |
| 163 |
sock.shutdown(socket.SHUT_RDWR) |
| 164 |
except OSError: |
| 165 |
pass |
| 166 |
sock.close() |
| 167 |
if return_socket is not None and family == return_family: |
| 168 |
return_socket.append(sock) |
| 169 |
break # Final iteration due to pre-loop logic; don't close. |
| 170 |
return port if got_socket else None |
| 171 |
|
| 172 |
|
| 173 |
def is_port_free(port): |
| 174 |
"""Check if specified port is free. |
| 175 |
|
| 176 |
Args: |
| 177 |
port: integer, port to check |
| 178 |
|
| 179 |
Returns: |
| 180 |
bool, whether port is free to use for both TCP and UDP. |
| 181 |
""" |
| 182 |
return _is_port_free(port) |
| 183 |
|
| 184 |
|
| 185 |
def _is_port_free(port, return_sockets=None): |
| 186 |
"""Internal implementation of is_port_free. |
| 187 |
|
| 188 |
Args: |
| 189 |
port: integer, port to check |
| 190 |
return_sockets: If supplied, a list that we will append open bound |
| 191 |
sockets on the port in question to rather than closing them. |
| 192 |
|
| 193 |
Returns: |
| 194 |
bool, whether port is free to use for both TCP and UDP. |
| 195 |
""" |
| 196 |
return (_bind(port, *_PROTOS[0], return_socket=return_sockets) and |
| 197 |
_bind(port, *_PROTOS[1], return_socket=return_sockets)) |
| 198 |
|
| 199 |
|
| 200 |
def pick_unused_port(pid=None, portserver_address=None): |
| 201 |
"""Picks an unused port and reserves it for use by a given process id. |
| 202 |
|
| 203 |
Args: |
| 204 |
pid: PID to tell the portserver to associate the reservation with. If |
| 205 |
None, the current process's PID is used. |
| 206 |
portserver_address: The address (path) of a unix domain socket |
| 207 |
with which to connect to a portserver, a leading '@' |
| 208 |
character indicates an address in the "abstract namespace". OR |
| 209 |
On systems without socket.AF_UNIX, this is an AF_INET address. |
| 210 |
If None, or no port is returned by the portserver at the provided |
| 211 |
address, the environment will be checked for a PORTSERVER_ADDRESS |
| 212 |
variable. If that is not set, no port server will be used. |
| 213 |
|
| 214 |
If no portserver is used, no pid based reservation is managed by any |
| 215 |
central authority. Race conditions and duplicate assignments may occur. |
| 216 |
|
| 217 |
Returns: |
| 218 |
A port number that is unused on both TCP and UDP. |
| 219 |
|
| 220 |
Raises: |
| 221 |
NoFreePortFoundError: No free port could be found. |
| 222 |
""" |
| 223 |
return _pick_unused_port(pid, portserver_address) |
| 224 |
|
| 225 |
|
| 226 |
def _pick_unused_port(pid=None, portserver_address=None, |
| 227 |
noserver_bind_timeout=0): |
| 228 |
"""Internal implementation of pick_unused_port. |
| 229 |
|
| 230 |
Args: |
| 231 |
pid, portserver_address: See pick_unused_port(). |
| 232 |
noserver_bind_timeout: If no portserver was used, this is the number of |
| 233 |
seconds we will attempt to keep a child process around with the ports |
| 234 |
returned open and bound SO_REUSEADDR style to help avoid race condition |
| 235 |
port reuse. A non-zero value attempts os.fork(). Do not use it in a |
| 236 |
multithreaded process. |
| 237 |
""" |
| 238 |
try: # Instead of `if _free_ports:` to handle the race condition. |
| 239 |
port = _free_ports.pop() |
| 240 |
except KeyError: |
| 241 |
pass |
| 242 |
else: |
| 243 |
_owned_ports.add(port) |
| 244 |
return port |
| 245 |
# Provide access to the portserver on an opt-in basis. |
| 246 |
if portserver_address: |
| 247 |
port = get_port_from_port_server(portserver_address, pid=pid) |
| 248 |
if port: |
| 249 |
return port |
| 250 |
if 'PORTSERVER_ADDRESS' in os.environ: |
| 251 |
port = get_port_from_port_server(os.environ['PORTSERVER_ADDRESS'], |
| 252 |
pid=pid) |
| 253 |
if port: |
| 254 |
return port |
| 255 |
return _pick_unused_port_without_server(bind_timeout=noserver_bind_timeout) |
| 256 |
|
| 257 |
|
| 258 |
def _spawn_bound_port_holding_daemon(port, bound_sockets, timeout): |
| 259 |
"""If possible, fork()s a daemon process to hold bound_sockets open. |
| 260 |
|
| 261 |
Emits a warning to stderr if it cannot. |
| 262 |
|
| 263 |
Args: |
| 264 |
port: The port number the sockets are bound to (informational). |
| 265 |
bound_sockets: The list of bound sockets our child process will hold |
| 266 |
open. If the list is empty, no action is taken. |
| 267 |
timeout: A positive number of seconds the child should sleep for before |
| 268 |
closing the sockets and exiting. |
| 269 |
""" |
| 270 |
if bound_sockets and timeout > 0: |
| 271 |
try: |
| 272 |
fork_pid = os.fork() # This concept only works on POSIX. |
| 273 |
except Exception as err: # pylint: disable=broad-except |
| 274 |
print('WARNING: Cannot timeout unbinding close of port', port, |
| 275 |
' closing on exit. -', err, file=sys.stderr) |
| 276 |
else: |
| 277 |
if fork_pid == 0: |
| 278 |
# This child process inherits and holds bound_sockets open |
| 279 |
# for bind_timeout seconds. |
| 280 |
try: |
| 281 |
# Close the stdio fds as may be connected to |
| 282 |
# a pipe that will cause a grandparent process |
| 283 |
# to wait on before returning. (cl/427587550) |
| 284 |
os.close(sys.stdin.fileno()) |
| 285 |
os.close(sys.stdout.fileno()) |
| 286 |
os.close(sys.stderr.fileno()) |
| 287 |
time.sleep(timeout) |
| 288 |
for held_socket in bound_sockets: |
| 289 |
held_socket.close() |
| 290 |
finally: |
| 291 |
os._exit(0) |
| 292 |
|
| 293 |
|
| 294 |
def _pick_unused_port_without_server(bind_timeout=0): |
| 295 |
"""Pick an available network port without the help of a port server. |
| 296 |
|
| 297 |
This code ensures that the port is available on both TCP and UDP. |
| 298 |
|
| 299 |
This function is an implementation detail of PickUnusedPort(), and |
| 300 |
should not be called by code outside of this module. |
| 301 |
|
| 302 |
Args: |
| 303 |
bind_timeout: number of seconds to attempt to keep a child process |
| 304 |
process around bound SO_REUSEADDR style to the port. If we cannot |
| 305 |
do that we emit a warning to stderr. |
| 306 |
|
| 307 |
Returns: |
| 308 |
A port number that is unused on both TCP and UDP. |
| 309 |
|
| 310 |
Raises: |
| 311 |
NoFreePortFoundError: No free port could be found. |
| 312 |
""" |
| 313 |
# Next, try a few times to get an OS-assigned port. |
| 314 |
# Ambrose discovered that on the 2.6 kernel, calling Bind() on UDP socket |
| 315 |
# returns the same port over and over. So always try TCP first. |
| 316 |
port = None |
| 317 |
bound_sockets = [] if bind_timeout > 0 else None |
| 318 |
for _ in range(10): |
| 319 |
# Ask the OS for an unused port. |
| 320 |
port = _bind(0, socket.SOCK_STREAM, socket.IPPROTO_TCP, bound_sockets) |
| 321 |
# Check if this port is unused on the other protocol. |
| 322 |
if (port and port not in _random_ports and |
| 323 |
_bind(port, socket.SOCK_DGRAM, socket.IPPROTO_UDP, bound_sockets)): |
| 324 |
_random_ports.add(port) |
| 325 |
_spawn_bound_port_holding_daemon(port, bound_sockets, bind_timeout) |
| 326 |
return port |
| 327 |
if bound_sockets: |
| 328 |
for held_socket in bound_sockets: |
| 329 |
held_socket.close() |
| 330 |
del bound_sockets[:] |
| 331 |
|
| 332 |
# Try random ports as a last resort. |
| 333 |
rng = random.Random() |
| 334 |
for _ in range(10): |
| 335 |
port = int(rng.randrange(15000, 25000)) |
| 336 |
if port not in _random_ports: |
| 337 |
if _is_port_free(port, bound_sockets): |
| 338 |
_random_ports.add(port) |
| 339 |
_spawn_bound_port_holding_daemon( |
| 340 |
port, bound_sockets, bind_timeout) |
| 341 |
return port |
| 342 |
if bound_sockets: |
| 343 |
for held_socket in bound_sockets: |
| 344 |
held_socket.close() |
| 345 |
del bound_sockets[:] |
| 346 |
|
| 347 |
# Give up. |
| 348 |
raise NoFreePortFoundError() |
| 349 |
|
| 350 |
|
| 351 |
def _posix_get_port_from_port_server(portserver_address, pid): |
| 352 |
# An AF_UNIX address may start with a zero byte, in which case it is in the |
| 353 |
# "abstract namespace", and doesn't have any filesystem representation. |
| 354 |
# See 'man 7 unix' for details. |
| 355 |
# The convention is to write '@' in the address to represent this zero byte. |
| 356 |
if portserver_address[0] == '@': |
| 357 |
portserver_address = '\0' + portserver_address[1:] |
| 358 |
|
| 359 |
try: |
| 360 |
# Create socket. |
| 361 |
if hasattr(socket, 'AF_UNIX'): |
| 362 |
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) |
| 363 |
else: |
| 364 |
# fallback to AF_INET if this is not unix |
| 365 |
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) |
| 366 |
try: |
| 367 |
# Connect to portserver. |
| 368 |
sock.connect(portserver_address) |
| 369 |
|
| 370 |
# Write request. |
| 371 |
sock.sendall(('%d\n' % pid).encode('ascii')) |
| 372 |
|
| 373 |
# Read response. |
| 374 |
# 1K should be ample buffer space. |
| 375 |
return sock.recv(1024) |
| 376 |
finally: |
| 377 |
sock.close() |
| 378 |
except socket.error as error: |
| 379 |
print('Socket error when connecting to portserver:', error, |
| 380 |
file=sys.stderr) |
| 381 |
return None |
| 382 |
|
| 383 |
|
| 384 |
def _windows_get_port_from_port_server(portserver_address, pid): |
| 385 |
if portserver_address[0] == '@': |
| 386 |
portserver_address = '\\\\.\\pipe\\' + portserver_address[1:] |
| 387 |
|
| 388 |
try: |
| 389 |
handle = _winapi.CreateFile( |
| 390 |
portserver_address, |
| 391 |
_winapi.GENERIC_READ | _winapi.GENERIC_WRITE, |
| 392 |
0, |
| 393 |
0, |
| 394 |
_winapi.OPEN_EXISTING, |
| 395 |
0, |
| 396 |
0) |
| 397 |
|
| 398 |
_winapi.WriteFile(handle, ('%d\n' % pid).encode('ascii')) |
| 399 |
data, _ = _winapi.ReadFile(handle, 6, 0) |
| 400 |
return data |
| 401 |
except FileNotFoundError as error: |
| 402 |
print('File error when connecting to portserver:', error, |
| 403 |
file=sys.stderr) |
| 404 |
return None |
| 405 |
|
| 406 |
|
| 407 |
def get_port_from_port_server(portserver_address, pid=None): |
| 408 |
"""Request a free a port from a system-wide portserver. |
| 409 |
|
| 410 |
This follows a very simple portserver protocol: |
| 411 |
The request consists of our pid (in ASCII) followed by a newline. |
| 412 |
The response is a port number and a newline, 0 on failure. |
| 413 |
|
| 414 |
This function is an implementation detail of pick_unused_port(). |
| 415 |
It should not normally be called by code outside of this module. |
| 416 |
|
| 417 |
Args: |
| 418 |
portserver_address: The address (path) of a unix domain socket |
| 419 |
with which to connect to the portserver. A leading '@' |
| 420 |
character indicates an address in the "abstract namespace." |
| 421 |
On systems without socket.AF_UNIX, this is an AF_INET address. |
| 422 |
pid: The PID to tell the portserver to associate the reservation with. |
| 423 |
If None, the current process's PID is used. |
| 424 |
|
| 425 |
Returns: |
| 426 |
The port number on success or None on failure. |
| 427 |
""" |
| 428 |
if not portserver_address: |
| 429 |
return None |
| 430 |
|
| 431 |
if pid is None: |
| 432 |
pid = os.getpid() |
| 433 |
|
| 434 |
if _winapi: |
| 435 |
buf = _windows_get_port_from_port_server(portserver_address, pid) |
| 436 |
else: |
| 437 |
buf = _posix_get_port_from_port_server(portserver_address, pid) |
| 438 |
|
| 439 |
if buf is None: |
| 440 |
return None |
| 441 |
|
| 442 |
try: |
| 443 |
port = int(buf.split(b'\n')[0]) |
| 444 |
except ValueError: |
| 445 |
print('Portserver failed to find a port.', file=sys.stderr) |
| 446 |
return None |
| 447 |
_owned_ports.add(port) |
| 448 |
return port |
| 449 |
|
| 450 |
|
| 451 |
# Legacy APIs. |
| 452 |
# pylint: disable=invalid-name |
| 453 |
Bind = bind |
| 454 |
GetPortFromPortServer = get_port_from_port_server |
| 455 |
IsPortFree = is_port_free |
| 456 |
PickUnusedPort = pick_unused_port |
| 457 |
# pylint: enable=invalid-name |
| 458 |
|
| 459 |
|
| 460 |
def main(argv): |
| 461 |
"""If passed an arg, treat it as a PID, otherwise we use getppid(). |
| 462 |
|
| 463 |
A second optional argument can be a bind timeout in seconds that will be |
| 464 |
used ONLY if no portserver is found. We attempt to leave a process around |
| 465 |
holding the port open and bound with SO_REUSEADDR set for timeout seconds. |
| 466 |
If the timeout bind was not possible, a warning is emitted to stderr. |
| 467 |
|
| 468 |
#!/bin/bash |
| 469 |
port="$(python -m portpicker $$ 1.23)" |
| 470 |
test_my_server "$port" |
| 471 |
|
| 472 |
This will pick a port for your script's PID and assign it to $port, if no |
| 473 |
portserver was used, it attempts to keep a socket bound to $port for 1.23 |
| 474 |
seconds after the portpicker process has exited. This is a convenient hack |
| 475 |
to attempt to prevent port reallocation during scripts outside of |
| 476 |
portserver managed environments. |
| 477 |
|
| 478 |
Older versions of the portpicker CLI ignore everything beyond the first arg. |
| 479 |
Older versions also used getpid() instead of getppid(), so script users are |
| 480 |
strongly encouraged to be explicit and pass $$ or your languages equivalent |
| 481 |
to associate the port with the PID of the controlling process. |
| 482 |
""" |
| 483 |
# Our command line is trivial so I avoid an argparse import. If we ever |
| 484 |
# grow more than 1-2 args, switch to a using argparse. |
| 485 |
if '-h' in argv or '--help' in argv: |
| 486 |
print(argv[0], 'usage:\n') |
| 487 |
import inspect |
| 488 |
print(inspect.getdoc(main)) |
| 489 |
sys.exit(1) |
| 490 |
pid=int(argv[1]) if len(argv) > 1 else os.getppid() |
| 491 |
bind_timeout=float(argv[2]) if len(argv) > 2 else 0 |
| 492 |
port = _pick_unused_port(pid=pid, noserver_bind_timeout=bind_timeout) |
| 493 |
if not port: |
| 494 |
sys.exit(1) |
| 495 |
print(port) |
| 496 |
|
| 497 |
|
| 498 |
if __name__ == '__main__': |
| 499 |
main(sys.argv) |