Debian Codegraph @

Lib/multiprocessing/shared_memory.py

raw
1 """Provides shared memory for direct access across processes.
2
3 The API of this package is currently provisional. Refer to the
4 documentation for details.
5 """
6
7
8 __all__ = [ 'SharedMemory', 'ShareableList' ]
9
10
11 from functools import partial
12 import mmap
13 import os
14 import errno
15 import struct
16 import secrets
17 import types
18
19 if os.name == "nt":
20 import _winapi
21 _USE_POSIX = False
22 else:
23 import _posixshmem
24 _USE_POSIX = True
25
26 from . import resource_tracker
27
28 _O_CREX = os.O_CREAT | os.O_EXCL
29
30 # FreeBSD (and perhaps other BSDs) limit names to 14 characters.
31 _SHM_SAFE_NAME_LENGTH = 14
32
33 # Shared memory block name prefix
34 if _USE_POSIX:
35 _SHM_NAME_PREFIX = '/psm_'
36 else:
37 _SHM_NAME_PREFIX = 'wnsm_'
38
39
40 def _make_filename():
41 "Create a random filename for the shared memory object."
42 # number of random bytes to use for name
43 nbytes = (_SHM_SAFE_NAME_LENGTH - len(_SHM_NAME_PREFIX)) // 2
44 assert nbytes >= 2, '_SHM_NAME_PREFIX too long'
45 name = _SHM_NAME_PREFIX + secrets.token_hex(nbytes)
46 assert len(name) <= _SHM_SAFE_NAME_LENGTH
47 return name
48
49
50 class SharedMemory:
51 """Creates a new shared memory block or attaches to an existing
52 shared memory block.
53
54 Every shared memory block is assigned a unique name. This enables
55 one process to create a shared memory block with a particular name
56 so that a different process can attach to that same shared memory
57 block using that same name.
58
59 As a resource for sharing data across processes, shared memory blocks
60 may outlive the original process that created them. When one process
61 no longer needs access to a shared memory block that might still be
62 needed by other processes, the close() method should be called.
63 When a shared memory block is no longer needed by any process, the
64 unlink() method should be called to ensure proper cleanup."""
65
66 # Defaults; enables close() and unlink() to run without errors.
67 _name = None
68 _fd = -1
69 _mmap = None
70 _buf = None
71 _flags = os.O_RDWR
72 _mode = 0o600
73 _prepend_leading_slash = True if _USE_POSIX else False
74 _track = True
75
76 def __init__(self, name=None, create=False, size=0, *, track=True):
77 if not size >= 0:
78 raise ValueError("'size' must be a positive integer")
79 if create:
80 self._flags = _O_CREX | os.O_RDWR
81 if size == 0:
82 raise ValueError("'size' must be a positive number different from zero")
83 if name is None and not self._flags & os.O_EXCL:
84 raise ValueError("'name' can only be None if create=True")
85
86 self._track = track
87 if _USE_POSIX:
88
89 # POSIX Shared Memory
90
91 if name is None:
92 while True:
93 name = _make_filename()
94 try:
95 self._fd = _posixshmem.shm_open(
96 name,
97 self._flags,
98 mode=self._mode
99 )
100 except FileExistsError:
101 continue
102 self._name = name
103 break
104 else:
105 name = "/" + name if self._prepend_leading_slash else name
106 self._fd = _posixshmem.shm_open(
107 name,
108 self._flags,
109 mode=self._mode
110 )
111 self._name = name
112 try:
113 if create and size:
114 os.ftruncate(self._fd, size)
115 stats = os.fstat(self._fd)
116 size = stats.st_size
117 self._mmap = mmap.mmap(self._fd, size)
118 except OSError:
119 self.unlink()
120 raise
121 if self._track:
122 resource_tracker.register(self._name, "shared_memory")
123
124 else:
125
126 # Windows Named Shared Memory
127
128 if create:
129 while True:
130 temp_name = _make_filename() if name is None else name
131 # Create and reserve shared memory block with this name
132 # until it can be attached to by mmap.
133 h_map = _winapi.CreateFileMapping(
134 _winapi.INVALID_HANDLE_VALUE,
135 _winapi.NULL,
136 _winapi.PAGE_READWRITE,
137 (size >> 32) & 0xFFFFFFFF,
138 size & 0xFFFFFFFF,
139 temp_name
140 )
141 try:
142 last_error_code = _winapi.GetLastError()
143 if last_error_code == _winapi.ERROR_ALREADY_EXISTS:
144 if name is not None:
145 raise FileExistsError(
146 errno.EEXIST,
147 os.strerror(errno.EEXIST),
148 name,
149 _winapi.ERROR_ALREADY_EXISTS
150 )
151 else:
152 continue
153 self._mmap = mmap.mmap(-1, size, tagname=temp_name)
154 finally:
155 _winapi.CloseHandle(h_map)
156 self._name = temp_name
157 break
158
159 else:
160 self._name = name
161 # Dynamically determine the existing named shared memory
162 # block's size which is likely a multiple of mmap.PAGESIZE.
163 h_map = _winapi.OpenFileMapping(
164 _winapi.FILE_MAP_READ,
165 False,
166 name
167 )
168 try:
169 p_buf = _winapi.MapViewOfFile(
170 h_map,
171 _winapi.FILE_MAP_READ,
172 0,
173 0,
174 0
175 )
176 finally:
177 _winapi.CloseHandle(h_map)
178 try:
179 size = _winapi.VirtualQuerySize(p_buf)
180 finally:
181 _winapi.UnmapViewOfFile(p_buf)
182 self._mmap = mmap.mmap(-1, size, tagname=name)
183
184 self._size = size
185 self._buf = memoryview(self._mmap)
186
187 def __del__(self):
188 try:
189 self.close()
190 except OSError:
191 pass
192
193 def __reduce__(self):
194 return (
195 self.__class__,
196 (
197 self.name,
198 False,
199 self.size,
200 ),
201 )
202
203 def __repr__(self):
204 return f'{self.__class__.__name__}({self.name!r}, size={self.size})'
205
206 @property
207 def buf(self):
208 "A memoryview of contents of the shared memory block."
209 return self._buf
210
211 @property
212 def name(self):
213 "Unique name that identifies the shared memory block."
214 reported_name = self._name
215 if _USE_POSIX and self._prepend_leading_slash:
216 if self._name.startswith("/"):
217 reported_name = self._name[1:]
218 return reported_name
219
220 @property
221 def size(self):
222 "Size in bytes."
223 return self._size
224
225 def close(self):
226 """Closes access to the shared memory from this instance but does
227 not destroy the shared memory block."""
228 if self._buf is not None:
229 self._buf.release()
230 self._buf = None
231 if self._mmap is not None:
232 self._mmap.close()
233 self._mmap = None
234 if _USE_POSIX and self._fd >= 0:
235 os.close(self._fd)
236 self._fd = -1
237
238 def unlink(self):
239 """Requests that the underlying shared memory block be destroyed.
240
241 Unlink should be called once (and only once) across all handles
242 which have access to the shared memory block, even if these
243 handles belong to different processes. Closing and unlinking may
244 happen in any order, but trying to access data inside a shared
245 memory block after unlinking may result in memory errors,
246 depending on platform.
247
248 This method has no effect on Windows, where the only way to
249 delete a shared memory block is to close all handles."""
250
251 if _USE_POSIX and self._name:
252 _posixshmem.shm_unlink(self._name)
253 if self._track:
254 resource_tracker.unregister(self._name, "shared_memory")
255
256
257 _encoding = "utf8"
258
259 class ShareableList:
260 """Pattern for a mutable list-like object shareable via a shared
261 memory block. It differs from the built-in list type in that these
262 lists can not change their overall length (i.e. no append, insert,
263 etc.)
264
265 Because values are packed into a memoryview as bytes, the struct
266 packing format for any storable value must require no more than 8
267 characters to describe its format."""
268
269 # The shared memory area is organized as follows:
270 # - 8 bytes: number of items (N) as a 64-bit integer
271 # - (N + 1) * 8 bytes: offsets of each element from the start of the
272 # data area
273 # - K bytes: the data area storing item values (with encoding and size
274 # depending on their respective types)
275 # - N * 8 bytes: `struct` format string for each element
276 # - N bytes: index into _back_transforms_mapping for each element
277 # (for reconstructing the corresponding Python value)
278 _types_mapping = {
279 int: "q",
280 float: "d",
281 bool: "xxxxxxx?",
282 str: "%ds",
283 bytes: "%ds",
284 None.__class__: "xxxxxx?x",
285 }
286 _alignment = 8
287 _back_transforms_mapping = {
288 0: lambda value: value, # int, float, bool
289 1: lambda value: value.rstrip(b'\x00').decode(_encoding), # str
290 2: lambda value: value.rstrip(b'\x00'), # bytes
291 3: lambda _value: None, # None
292 }
293
294 @staticmethod
295 def _extract_recreation_code(value):
296 """Used in concert with _back_transforms_mapping to convert values
297 into the appropriate Python objects when retrieving them from
298 the list as well as when storing them."""
299 if not isinstance(value, (str, bytes, None.__class__)):
300 return 0
301 elif isinstance(value, str):
302 return 1
303 elif isinstance(value, bytes):
304 return 2
305 else:
306 return 3 # NoneType
307
308 def __init__(self, sequence=None, *, name=None):
309 if name is None or sequence is not None:
310 sequence = sequence or ()
311 _formats = [
312 self._types_mapping[type(item)]
313 if not isinstance(item, (str, bytes))
314 else self._types_mapping[type(item)] % (
315 self._alignment * (len(item) // self._alignment + 1),
316 )
317 for item in sequence
318 ]
319 self._list_len = len(_formats)
320 assert sum(len(fmt) <= 8 for fmt in _formats) == self._list_len
321 offset = 0
322 # The offsets of each list element into the shared memory's
323 # data area (0 meaning the start of the data area, not the start
324 # of the shared memory area).
325 self._allocated_offsets = [0]
326 for fmt in _formats:
327 offset += self._alignment if fmt[-1] != "s" else int(fmt[:-1])
328 self._allocated_offsets.append(offset)
329 _recreation_codes = [
330 self._extract_recreation_code(item) for item in sequence
331 ]
332 requested_size = struct.calcsize(
333 "q" + self._format_size_metainfo +
334 "".join(_formats) +
335 self._format_packing_metainfo +
336 self._format_back_transform_codes
337 )
338
339 self.shm = SharedMemory(name, create=True, size=requested_size)
340 else:
341 self.shm = SharedMemory(name)
342
343 if sequence is not None:
344 _enc = _encoding
345 struct.pack_into(
346 "q" + self._format_size_metainfo,
347 self.shm.buf,
348 0,
349 self._list_len,
350 *(self._allocated_offsets)
351 )
352 struct.pack_into(
353 "".join(_formats),
354 self.shm.buf,
355 self._offset_data_start,
356 *(v.encode(_enc) if isinstance(v, str) else v for v in sequence)
357 )
358 struct.pack_into(
359 self._format_packing_metainfo,
360 self.shm.buf,
361 self._offset_packing_formats,
362 *(v.encode(_enc) for v in _formats)
363 )
364 struct.pack_into(
365 self._format_back_transform_codes,
366 self.shm.buf,
367 self._offset_back_transform_codes,
368 *(_recreation_codes)
369 )
370
371 else:
372 self._list_len = len(self) # Obtains size from offset 0 in buffer.
373 self._allocated_offsets = list(
374 struct.unpack_from(
375 self._format_size_metainfo,
376 self.shm.buf,
377 1 * 8
378 )
379 )
380
381 def _get_packing_format(self, position):
382 "Gets the packing format for a single value stored in the list."
383 position = position if position >= 0 else position + self._list_len
384 if (position >= self._list_len) or (self._list_len < 0):
385 raise IndexError("Requested position out of range.")
386
387 v = struct.unpack_from(
388 "8s",
389 self.shm.buf,
390 self._offset_packing_formats + position * 8
391 )[0]
392 fmt = v.rstrip(b'\x00')
393 fmt_as_str = fmt.decode(_encoding)
394
395 return fmt_as_str
396
397 def _get_back_transform(self, position):
398 "Gets the back transformation function for a single value."
399
400 if (position >= self._list_len) or (self._list_len < 0):
401 raise IndexError("Requested position out of range.")
402
403 transform_code = struct.unpack_from(
404 "b",
405 self.shm.buf,
406 self._offset_back_transform_codes + position
407 )[0]
408 transform_function = self._back_transforms_mapping[transform_code]
409
410 return transform_function
411
412 def _set_packing_format_and_transform(self, position, fmt_as_str, value):
413 """Sets the packing format and back transformation code for a
414 single value in the list at the specified position."""
415
416 if (position >= self._list_len) or (self._list_len < 0):
417 raise IndexError("Requested position out of range.")
418
419 struct.pack_into(
420 "8s",
421 self.shm.buf,
422 self._offset_packing_formats + position * 8,
423 fmt_as_str.encode(_encoding)
424 )
425
426 transform_code = self._extract_recreation_code(value)
427 struct.pack_into(
428 "b",
429 self.shm.buf,
430 self._offset_back_transform_codes + position,
431 transform_code
432 )
433
434 def __getitem__(self, position):
435 position = position if position >= 0 else position + self._list_len
436 try:
437 offset = self._offset_data_start + self._allocated_offsets[position]
438 (v,) = struct.unpack_from(
439 self._get_packing_format(position),
440 self.shm.buf,
441 offset
442 )
443 except IndexError:
444 raise IndexError("index out of range")
445
446 back_transform = self._get_back_transform(position)
447 v = back_transform(v)
448
449 return v
450
451 def __setitem__(self, position, value):
452 position = position if position >= 0 else position + self._list_len
453 try:
454 item_offset = self._allocated_offsets[position]
455 offset = self._offset_data_start + item_offset
456 current_format = self._get_packing_format(position)
457 except IndexError:
458 raise IndexError("assignment index out of range")
459
460 if not isinstance(value, (str, bytes)):
461 new_format = self._types_mapping[type(value)]
462 encoded_value = value
463 else:
464 allocated_length = self._allocated_offsets[position + 1] - item_offset
465
466 encoded_value = (value.encode(_encoding)
467 if isinstance(value, str) else value)
468 if len(encoded_value) > allocated_length:
469 raise ValueError("bytes/str item exceeds available storage")
470 if current_format[-1] == "s":
471 new_format = current_format
472 else:
473 new_format = self._types_mapping[str] % (
474 allocated_length,
475 )
476
477 self._set_packing_format_and_transform(
478 position,
479 new_format,
480 value
481 )
482 struct.pack_into(new_format, self.shm.buf, offset, encoded_value)
483
484 def __reduce__(self):
485 return partial(self.__class__, name=self.shm.name), ()
486
487 def __len__(self):
488 return struct.unpack_from("q", self.shm.buf, 0)[0]
489
490 def __repr__(self):
491 return f'{self.__class__.__name__}({list(self)}, name={self.shm.name!r})'
492
493 @property
494 def format(self):
495 "The struct packing format used by all currently stored items."
496 return "".join(
497 self._get_packing_format(i) for i in range(self._list_len)
498 )
499
500 @property
501 def _format_size_metainfo(self):
502 "The struct packing format used for the items' storage offsets."
503 return "q" * (self._list_len + 1)
504
505 @property
506 def _format_packing_metainfo(self):
507 "The struct packing format used for the items' packing formats."
508 return "8s" * self._list_len
509
510 @property
511 def _format_back_transform_codes(self):
512 "The struct packing format used for the items' back transforms."
513 return "b" * self._list_len
514
515 @property
516 def _offset_data_start(self):
517 # - 8 bytes for the list length
518 # - (N + 1) * 8 bytes for the element offsets
519 return (self._list_len + 2) * 8
520
521 @property
522 def _offset_packing_formats(self):
523 return self._offset_data_start + self._allocated_offsets[-1]
524
525 @property
526 def _offset_back_transform_codes(self):
527 return self._offset_packing_formats + self._list_len * 8
528
529 def count(self, value):
530 "L.count(value) -> integer -- return number of occurrences of value."
531
532 return sum(value == entry for entry in self)
533
534 def index(self, value):
535 """L.index(value) -> integer -- return first index of value.
536 Raises ValueError if the value is not present."""
537
538 for position, entry in enumerate(self):
539 if value == entry:
540 return position
541 else:
542 raise ValueError("ShareableList.index(x): x not in list")
543
544 __class_getitem__ = classmethod(types.GenericAlias)