| 1 |
"""Python implementation of computing the layout of a struct/union |
| 2 |
|
| 3 |
This code is internal and tightly coupled to the C part. The interface |
| 4 |
may change at any time. |
| 5 |
""" |
| 6 |
|
| 7 |
import sys |
| 8 |
import warnings |
| 9 |
|
| 10 |
from _ctypes import CField, buffer_info |
| 11 |
import ctypes |
| 12 |
|
| 13 |
def round_down(n, multiple): |
| 14 |
assert n >= 0 |
| 15 |
assert multiple > 0 |
| 16 |
return (n // multiple) * multiple |
| 17 |
|
| 18 |
def round_up(n, multiple): |
| 19 |
assert n >= 0 |
| 20 |
assert multiple > 0 |
| 21 |
return ((n + multiple - 1) // multiple) * multiple |
| 22 |
|
| 23 |
_INT_MAX = (1 << (ctypes.sizeof(ctypes.c_int) * 8) - 1) - 1 |
| 24 |
|
| 25 |
|
| 26 |
class StructUnionLayout: |
| 27 |
def __init__(self, fields, size, align, format_spec): |
| 28 |
# sequence of CField objects |
| 29 |
self.fields = fields |
| 30 |
|
| 31 |
# total size of the aggregate (rounded up to alignment) |
| 32 |
self.size = size |
| 33 |
|
| 34 |
# total alignment requirement of the aggregate |
| 35 |
self.align = align |
| 36 |
|
| 37 |
# buffer format specification (as a string, UTF-8 but bes |
| 38 |
# kept ASCII-only) |
| 39 |
self.format_spec = format_spec |
| 40 |
|
| 41 |
|
| 42 |
def get_layout(cls, input_fields, is_struct, base): |
| 43 |
"""Return a StructUnionLayout for the given class. |
| 44 |
|
| 45 |
Called by PyCStructUnionType_update_stginfo when _fields_ is assigned |
| 46 |
to a class. |
| 47 |
""" |
| 48 |
# Currently there are two modes, selectable using the '_layout_' attribute: |
| 49 |
# |
| 50 |
# 'gcc-sysv' mode places fields one after another, bit by bit. |
| 51 |
# But "each bit field must fit within a single object of its specified |
| 52 |
# type" (GCC manual, section 15.8 "Bit Field Packing"). When it doesn't, |
| 53 |
# we insert a few bits of padding to avoid that. |
| 54 |
# |
| 55 |
# 'ms' mode works similar except for bitfield packing. Adjacent |
| 56 |
# bit-fields are packed into the same 1-, 2-, or 4-byte allocation unit |
| 57 |
# if the integral types are the same size and if the next bit-field fits |
| 58 |
# into the current allocation unit without crossing the boundary imposed |
| 59 |
# by the common alignment requirements of the bit-fields. |
| 60 |
# |
| 61 |
# See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html#index-mms-bitfields |
| 62 |
# for details. |
| 63 |
|
| 64 |
# We do not support zero length bitfields (we use bitsize != 0 |
| 65 |
# elsewhere to indicate a bitfield). Here, non-bitfields have bit_size |
| 66 |
# set to size*8. |
| 67 |
|
| 68 |
# For clarity, variables that count bits have `bit` in their names. |
| 69 |
|
| 70 |
pack = getattr(cls, '_pack_', None) |
| 71 |
|
| 72 |
layout = getattr(cls, '_layout_', None) |
| 73 |
if layout is None: |
| 74 |
if sys.platform == 'win32': |
| 75 |
gcc_layout = False |
| 76 |
elif pack: |
| 77 |
if is_struct: |
| 78 |
base_type_name = 'Structure' |
| 79 |
else: |
| 80 |
base_type_name = 'Union' |
| 81 |
warnings._deprecated( |
| 82 |
'_pack_ without _layout_', |
| 83 |
f"Due to '_pack_', the '{cls.__name__}' {base_type_name} will " |
| 84 |
+ "use memory layout compatible with MSVC (Windows). " |
| 85 |
+ "If this is intended, set _layout_ to 'ms'. " |
| 86 |
+ "The implicit default is deprecated and slated to become " |
| 87 |
+ "an error in Python {remove}.", |
| 88 |
remove=(3, 19), |
| 89 |
) |
| 90 |
gcc_layout = False |
| 91 |
else: |
| 92 |
gcc_layout = True |
| 93 |
elif layout == 'ms': |
| 94 |
gcc_layout = False |
| 95 |
elif layout == 'gcc-sysv': |
| 96 |
gcc_layout = True |
| 97 |
else: |
| 98 |
raise ValueError(f'unknown _layout_: {layout!r}') |
| 99 |
|
| 100 |
align = getattr(cls, '_align_', 1) |
| 101 |
if align < 0: |
| 102 |
raise ValueError('_align_ must be a non-negative integer') |
| 103 |
elif align == 0: |
| 104 |
# Setting `_align_ = 0` amounts to using the default alignment |
| 105 |
align = 1 |
| 106 |
|
| 107 |
if base: |
| 108 |
align = max(ctypes.alignment(base), align) |
| 109 |
|
| 110 |
swapped_bytes = hasattr(cls, '_swappedbytes_') |
| 111 |
if swapped_bytes: |
| 112 |
big_endian = sys.byteorder == 'little' |
| 113 |
else: |
| 114 |
big_endian = sys.byteorder == 'big' |
| 115 |
|
| 116 |
if pack is not None: |
| 117 |
try: |
| 118 |
pack = int(pack) |
| 119 |
except (TypeError, ValueError): |
| 120 |
raise ValueError("_pack_ must be an integer") |
| 121 |
if pack < 0: |
| 122 |
raise ValueError("_pack_ must be a non-negative integer") |
| 123 |
if pack > _INT_MAX: |
| 124 |
raise ValueError("_pack_ too big") |
| 125 |
if gcc_layout: |
| 126 |
raise ValueError('_pack_ is not compatible with gcc-sysv layout') |
| 127 |
|
| 128 |
result_fields = [] |
| 129 |
|
| 130 |
if is_struct: |
| 131 |
format_spec_parts = ["T{"] |
| 132 |
else: |
| 133 |
format_spec_parts = ["B"] |
| 134 |
|
| 135 |
last_field_bit_size = 0 # used in MS layout only |
| 136 |
|
| 137 |
# `8 * next_byte_offset + next_bit_offset` points to where the |
| 138 |
# next field would start. |
| 139 |
next_bit_offset = 0 |
| 140 |
next_byte_offset = 0 |
| 141 |
|
| 142 |
# size if this was a struct (sum of field sizes, plus padding) |
| 143 |
struct_size = 0 |
| 144 |
# max of field sizes; only meaningful for unions |
| 145 |
union_size = 0 |
| 146 |
|
| 147 |
if base: |
| 148 |
struct_size = ctypes.sizeof(base) |
| 149 |
if gcc_layout: |
| 150 |
next_bit_offset = struct_size * 8 |
| 151 |
else: |
| 152 |
next_byte_offset = struct_size |
| 153 |
|
| 154 |
last_size = struct_size |
| 155 |
for i, field in enumerate(input_fields): |
| 156 |
if not is_struct: |
| 157 |
# Unions start fresh each time |
| 158 |
last_field_bit_size = 0 |
| 159 |
next_bit_offset = 0 |
| 160 |
next_byte_offset = 0 |
| 161 |
|
| 162 |
# Unpack the field |
| 163 |
field = tuple(field) |
| 164 |
try: |
| 165 |
name, ctype = field |
| 166 |
except (ValueError, TypeError): |
| 167 |
try: |
| 168 |
name, ctype, bit_size = field |
| 169 |
except (ValueError, TypeError) as exc: |
| 170 |
raise ValueError( |
| 171 |
'_fields_ must be a sequence of (name, C type) pairs ' |
| 172 |
+ 'or (name, C type, bit size) triples') from exc |
| 173 |
is_bitfield = True |
| 174 |
if bit_size <= 0: |
| 175 |
raise ValueError( |
| 176 |
f'number of bits invalid for bit field {name!r}') |
| 177 |
type_size = ctypes.sizeof(ctype) |
| 178 |
if bit_size > type_size * 8: |
| 179 |
raise ValueError( |
| 180 |
f'number of bits invalid for bit field {name!r}') |
| 181 |
else: |
| 182 |
is_bitfield = False |
| 183 |
type_size = ctypes.sizeof(ctype) |
| 184 |
bit_size = type_size * 8 |
| 185 |
|
| 186 |
type_bit_size = type_size * 8 |
| 187 |
type_align = ctypes.alignment(ctype) or 1 |
| 188 |
type_bit_align = type_align * 8 |
| 189 |
|
| 190 |
if gcc_layout: |
| 191 |
# We don't use next_byte_offset here |
| 192 |
assert pack is None |
| 193 |
assert next_byte_offset == 0 |
| 194 |
|
| 195 |
# Determine whether the bit field, if placed at the next |
| 196 |
# free bit, fits within a single object of its specified type. |
| 197 |
# That is: determine a "slot", sized & aligned for the |
| 198 |
# specified type, which contains the bitfield's beginning: |
| 199 |
slot_start_bit = round_down(next_bit_offset, type_bit_align) |
| 200 |
slot_end_bit = slot_start_bit + type_bit_size |
| 201 |
# And see if it also contains the bitfield's last bit: |
| 202 |
field_end_bit = next_bit_offset + bit_size |
| 203 |
if field_end_bit > slot_end_bit: |
| 204 |
# It doesn't: add padding (bump up to the next |
| 205 |
# alignment boundary) |
| 206 |
next_bit_offset = round_up(next_bit_offset, type_bit_align) |
| 207 |
|
| 208 |
offset = round_down(next_bit_offset, type_bit_align) // 8 |
| 209 |
if is_bitfield: |
| 210 |
bit_offset = next_bit_offset - 8 * offset |
| 211 |
assert bit_offset <= type_bit_size |
| 212 |
else: |
| 213 |
assert offset == next_bit_offset / 8 |
| 214 |
|
| 215 |
next_bit_offset += bit_size |
| 216 |
struct_size = round_up(next_bit_offset, 8) // 8 |
| 217 |
else: |
| 218 |
if pack: |
| 219 |
type_align = min(pack, type_align) |
| 220 |
|
| 221 |
# next_byte_offset points to end of current bitfield. |
| 222 |
# next_bit_offset is generally non-positive, |
| 223 |
# and 8 * next_byte_offset + next_bit_offset points just behind |
| 224 |
# the end of the last field we placed. |
| 225 |
if ( |
| 226 |
(0 < next_bit_offset + bit_size) |
| 227 |
or (type_bit_size != last_field_bit_size) |
| 228 |
): |
| 229 |
# Close the previous bitfield (if any) |
| 230 |
# and start a new bitfield |
| 231 |
next_byte_offset = round_up(next_byte_offset, type_align) |
| 232 |
|
| 233 |
next_byte_offset += type_size |
| 234 |
|
| 235 |
last_field_bit_size = type_bit_size |
| 236 |
# Reminder: 8 * (next_byte_offset) + next_bit_offset |
| 237 |
# points to where we would start a new field, namely |
| 238 |
# just behind where we placed the last field plus an |
| 239 |
# allowance for alignment. |
| 240 |
next_bit_offset = -last_field_bit_size |
| 241 |
|
| 242 |
assert type_bit_size == last_field_bit_size |
| 243 |
|
| 244 |
offset = next_byte_offset - last_field_bit_size // 8 |
| 245 |
if is_bitfield: |
| 246 |
assert 0 <= (last_field_bit_size + next_bit_offset) |
| 247 |
bit_offset = last_field_bit_size + next_bit_offset |
| 248 |
if type_bit_size: |
| 249 |
assert (last_field_bit_size + next_bit_offset) < type_bit_size |
| 250 |
|
| 251 |
next_bit_offset += bit_size |
| 252 |
struct_size = next_byte_offset |
| 253 |
|
| 254 |
if is_bitfield and big_endian: |
| 255 |
# On big-endian architectures, bit fields are also laid out |
| 256 |
# starting with the big end. |
| 257 |
bit_offset = type_bit_size - bit_size - bit_offset |
| 258 |
|
| 259 |
# Add the format spec parts |
| 260 |
if is_struct: |
| 261 |
padding = offset - last_size |
| 262 |
format_spec_parts.append(padding_spec(padding)) |
| 263 |
|
| 264 |
fieldfmt, bf_ndim, bf_shape = buffer_info(ctype) |
| 265 |
|
| 266 |
if bf_shape: |
| 267 |
format_spec_parts.extend(( |
| 268 |
"(", |
| 269 |
','.join(str(n) for n in bf_shape), |
| 270 |
")", |
| 271 |
)) |
| 272 |
|
| 273 |
if fieldfmt is None: |
| 274 |
fieldfmt = "B" |
| 275 |
if isinstance(name, bytes): |
| 276 |
# a bytes name would be rejected later, but we check early |
| 277 |
# to avoid a BytesWarning with `python -bb` |
| 278 |
raise TypeError( |
| 279 |
f"field {name!r}: name must be a string, not bytes") |
| 280 |
format_spec_parts.append(f"{fieldfmt}:{name}:") |
| 281 |
|
| 282 |
result_fields.append(CField( |
| 283 |
name=name, |
| 284 |
type=ctype, |
| 285 |
byte_size=type_size, |
| 286 |
byte_offset=offset, |
| 287 |
bit_size=bit_size if is_bitfield else None, |
| 288 |
bit_offset=bit_offset if is_bitfield else None, |
| 289 |
index=i, |
| 290 |
|
| 291 |
# Do not use CField outside ctypes, yet. |
| 292 |
# The constructor is internal API and may change without warning. |
| 293 |
_internal_use=True, |
| 294 |
)) |
| 295 |
if is_bitfield and not gcc_layout: |
| 296 |
assert type_bit_size > 0 |
| 297 |
|
| 298 |
align = max(align, type_align) |
| 299 |
last_size = struct_size |
| 300 |
if not is_struct: |
| 301 |
union_size = max(struct_size, union_size) |
| 302 |
|
| 303 |
if is_struct: |
| 304 |
total_size = struct_size |
| 305 |
else: |
| 306 |
total_size = union_size |
| 307 |
|
| 308 |
# Adjust the size according to the alignment requirements |
| 309 |
aligned_size = round_up(total_size, align) |
| 310 |
|
| 311 |
# Finish up the format spec |
| 312 |
if is_struct: |
| 313 |
padding = aligned_size - total_size |
| 314 |
format_spec_parts.append(padding_spec(padding)) |
| 315 |
format_spec_parts.append("}") |
| 316 |
|
| 317 |
return StructUnionLayout( |
| 318 |
fields=result_fields, |
| 319 |
size=aligned_size, |
| 320 |
align=align, |
| 321 |
format_spec="".join(format_spec_parts), |
| 322 |
) |
| 323 |
|
| 324 |
|
| 325 |
def padding_spec(padding): |
| 326 |
if padding <= 0: |
| 327 |
return "" |
| 328 |
if padding == 1: |
| 329 |
return "x" |
| 330 |
return f"{padding}x" |