| 1 |
""" |
| 2 |
Multi-part parsing for file uploads. |
| 3 |
|
| 4 |
Exposes one class, ``MultiPartParser``, which feeds chunks of uploaded data to |
| 5 |
file upload handlers for processing. |
| 6 |
""" |
| 7 |
|
| 8 |
import base64 |
| 9 |
import binascii |
| 10 |
import collections |
| 11 |
import html |
| 12 |
|
| 13 |
from django.conf import settings |
| 14 |
from django.core.exceptions import ( |
| 15 |
RequestDataTooBig, |
| 16 |
SuspiciousMultipartForm, |
| 17 |
TooManyFieldsSent, |
| 18 |
TooManyFilesSent, |
| 19 |
) |
| 20 |
from django.core.files.uploadhandler import SkipFile, StopFutureHandlers, StopUpload |
| 21 |
from django.utils.datastructures import MultiValueDict |
| 22 |
from django.utils.encoding import force_str |
| 23 |
from django.utils.http import parse_header_parameters |
| 24 |
from django.utils.regex_helper import _lazy_re_compile |
| 25 |
|
| 26 |
__all__ = ("MultiPartParser", "MultiPartParserError", "InputStreamExhausted") |
| 27 |
|
| 28 |
|
| 29 |
class MultiPartParserError(Exception): |
| 30 |
pass |
| 31 |
|
| 32 |
|
| 33 |
class InputStreamExhausted(Exception): |
| 34 |
""" |
| 35 |
No more reads are allowed from this device. |
| 36 |
""" |
| 37 |
|
| 38 |
pass |
| 39 |
|
| 40 |
|
| 41 |
RAW = "raw" |
| 42 |
FILE = "file" |
| 43 |
FIELD = "field" |
| 44 |
FIELD_TYPES = frozenset([FIELD, RAW]) |
| 45 |
MAX_TOTAL_HEADER_SIZE = 1024 |
| 46 |
|
| 47 |
|
| 48 |
class MultiPartParser: |
| 49 |
""" |
| 50 |
An RFC 7578 multipart/form-data parser. |
| 51 |
|
| 52 |
``MultiValueDict.parse()`` reads the input stream in ``chunk_size`` chunks |
| 53 |
and returns a tuple of ``(MultiValueDict(POST), MultiValueDict(FILES))``. |
| 54 |
""" |
| 55 |
|
| 56 |
boundary_re = _lazy_re_compile(r"[ -~]{0,200}[!-~]") |
| 57 |
|
| 58 |
def __init__(self, META, input_data, upload_handlers, encoding=None): |
| 59 |
""" |
| 60 |
Initialize the MultiPartParser object. |
| 61 |
|
| 62 |
:META: |
| 63 |
The standard ``META`` dictionary in Django request objects. |
| 64 |
:input_data: |
| 65 |
The raw post data, as a file-like object. |
| 66 |
:upload_handlers: |
| 67 |
A list of UploadHandler instances that perform operations on the |
| 68 |
uploaded data. |
| 69 |
:encoding: |
| 70 |
The encoding with which to treat the incoming data. |
| 71 |
""" |
| 72 |
# Content-Type should contain multipart and the boundary information. |
| 73 |
content_type = META.get("CONTENT_TYPE", "") |
| 74 |
if not content_type.startswith("multipart/"): |
| 75 |
raise MultiPartParserError("Invalid Content-Type: %s" % content_type) |
| 76 |
|
| 77 |
try: |
| 78 |
content_type.encode("ascii") |
| 79 |
except UnicodeEncodeError: |
| 80 |
raise MultiPartParserError( |
| 81 |
"Invalid non-ASCII Content-Type in multipart: %s" |
| 82 |
% force_str(content_type) |
| 83 |
) |
| 84 |
|
| 85 |
# Parse the header to get the boundary to split the parts. |
| 86 |
_, opts = parse_header_parameters(content_type) |
| 87 |
boundary = opts.get("boundary") |
| 88 |
if not boundary or not self.boundary_re.fullmatch(boundary): |
| 89 |
raise MultiPartParserError( |
| 90 |
"Invalid boundary in multipart: %s" % force_str(boundary) |
| 91 |
) |
| 92 |
|
| 93 |
# Content-Length should contain the length of the body we are about |
| 94 |
# to receive. |
| 95 |
try: |
| 96 |
content_length = int(META.get("CONTENT_LENGTH", 0)) |
| 97 |
except (ValueError, TypeError): |
| 98 |
content_length = 0 |
| 99 |
|
| 100 |
if content_length < 0: |
| 101 |
# This means we shouldn't continue...raise an error. |
| 102 |
raise MultiPartParserError("Invalid content length: %r" % content_length) |
| 103 |
|
| 104 |
self._boundary = boundary.encode("ascii") |
| 105 |
self._input_data = input_data |
| 106 |
|
| 107 |
# For compatibility with low-level network APIs (with 32-bit integers), |
| 108 |
# the chunk size should be < 2^31, but still divisible by 4. |
| 109 |
possible_sizes = [x.chunk_size for x in upload_handlers if x.chunk_size] |
| 110 |
self._chunk_size = min([2**31 - 4] + possible_sizes) |
| 111 |
|
| 112 |
self._meta = META |
| 113 |
self._encoding = encoding or settings.DEFAULT_CHARSET |
| 114 |
self._content_length = content_length |
| 115 |
self._upload_handlers = upload_handlers |
| 116 |
|
| 117 |
def parse(self): |
| 118 |
# Call the actual parse routine and close all open files in case of |
| 119 |
# errors. This is needed because if exceptions are thrown the |
| 120 |
# MultiPartParser will not be garbage collected immediately and |
| 121 |
# resources would be kept alive. This is only needed for errors because |
| 122 |
# the Request object closes all uploaded files at the end of the |
| 123 |
# request. |
| 124 |
try: |
| 125 |
return self._parse() |
| 126 |
except Exception: |
| 127 |
if hasattr(self, "_files"): |
| 128 |
for _, files in self._files.lists(): |
| 129 |
for fileobj in files: |
| 130 |
fileobj.close() |
| 131 |
raise |
| 132 |
|
| 133 |
def _parse(self): |
| 134 |
""" |
| 135 |
Parse the POST data and break it into a FILES MultiValueDict and a POST |
| 136 |
MultiValueDict. |
| 137 |
|
| 138 |
Return a tuple containing the POST and FILES dictionary, respectively. |
| 139 |
""" |
| 140 |
from django.http import QueryDict |
| 141 |
|
| 142 |
encoding = self._encoding |
| 143 |
handlers = self._upload_handlers |
| 144 |
|
| 145 |
# HTTP spec says that Content-Length >= 0 is valid |
| 146 |
# handling content-length == 0 before continuing |
| 147 |
if self._content_length == 0: |
| 148 |
return QueryDict(encoding=self._encoding), MultiValueDict() |
| 149 |
|
| 150 |
# See if any of the handlers take care of the parsing. |
| 151 |
# This allows overriding everything if need be. |
| 152 |
for handler in handlers: |
| 153 |
result = handler.handle_raw_input( |
| 154 |
self._input_data, |
| 155 |
self._meta, |
| 156 |
self._content_length, |
| 157 |
self._boundary, |
| 158 |
encoding, |
| 159 |
) |
| 160 |
# Check to see if it was handled |
| 161 |
if result is not None: |
| 162 |
return result[0], result[1] |
| 163 |
|
| 164 |
# Create the data structures to be used later. |
| 165 |
self._post = QueryDict(mutable=True) |
| 166 |
self._files = MultiValueDict() |
| 167 |
|
| 168 |
# Instantiate the parser and stream: |
| 169 |
stream = LazyStream(ChunkIter(self._input_data, self._chunk_size)) |
| 170 |
|
| 171 |
# Whether or not to signal a file-completion at the beginning of the loop. |
| 172 |
old_field_name = None |
| 173 |
counters = [0] * len(handlers) |
| 174 |
|
| 175 |
# Number of bytes that have been read. |
| 176 |
num_bytes_read = 0 |
| 177 |
# To count the number of keys in the request. |
| 178 |
num_post_keys = 0 |
| 179 |
# To count the number of files in the request. |
| 180 |
num_files = 0 |
| 181 |
# To limit the amount of data read from the request. |
| 182 |
read_size = None |
| 183 |
# Whether a file upload is finished. |
| 184 |
uploaded_file = True |
| 185 |
|
| 186 |
try: |
| 187 |
for item_type, meta_data, field_stream in Parser(stream, self._boundary): |
| 188 |
if old_field_name: |
| 189 |
# We run this at the beginning of the next loop |
| 190 |
# since we cannot be sure a file is complete until |
| 191 |
# we hit the next boundary/part of the multipart content. |
| 192 |
self.handle_file_complete(old_field_name, counters) |
| 193 |
old_field_name = None |
| 194 |
uploaded_file = True |
| 195 |
|
| 196 |
if ( |
| 197 |
item_type in FIELD_TYPES |
| 198 |
and settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None |
| 199 |
): |
| 200 |
# Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS. |
| 201 |
num_post_keys += 1 |
| 202 |
# 2 accounts for empty raw fields before and after the |
| 203 |
# last boundary. |
| 204 |
if settings.DATA_UPLOAD_MAX_NUMBER_FIELDS + 2 < num_post_keys: |
| 205 |
raise TooManyFieldsSent( |
| 206 |
"The number of GET/POST parameters exceeded " |
| 207 |
"settings.DATA_UPLOAD_MAX_NUMBER_FIELDS." |
| 208 |
) |
| 209 |
|
| 210 |
try: |
| 211 |
disposition = meta_data["content-disposition"][1] |
| 212 |
field_name = disposition["name"].strip() |
| 213 |
except (KeyError, IndexError, AttributeError): |
| 214 |
continue |
| 215 |
|
| 216 |
transfer_encoding = meta_data.get("content-transfer-encoding") |
| 217 |
if transfer_encoding is not None: |
| 218 |
transfer_encoding = transfer_encoding[0].strip() |
| 219 |
field_name = force_str(field_name, encoding, errors="replace") |
| 220 |
|
| 221 |
if item_type == FIELD: |
| 222 |
# Avoid reading more than DATA_UPLOAD_MAX_MEMORY_SIZE. |
| 223 |
if settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None: |
| 224 |
read_size = ( |
| 225 |
settings.DATA_UPLOAD_MAX_MEMORY_SIZE - num_bytes_read |
| 226 |
) |
| 227 |
|
| 228 |
# This is a post field, we can just set it in the post |
| 229 |
if transfer_encoding == "base64": |
| 230 |
raw_data = field_stream.read(size=read_size) |
| 231 |
num_bytes_read += len(raw_data) |
| 232 |
try: |
| 233 |
data = base64.b64decode(raw_data) |
| 234 |
except binascii.Error: |
| 235 |
data = raw_data |
| 236 |
else: |
| 237 |
data = field_stream.read(size=read_size) |
| 238 |
num_bytes_read += len(data) |
| 239 |
|
| 240 |
# Add two here to make the check consistent with the |
| 241 |
# x-www-form-urlencoded check that includes '&='. |
| 242 |
num_bytes_read += len(field_name) + 2 |
| 243 |
if ( |
| 244 |
settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None |
| 245 |
and num_bytes_read > settings.DATA_UPLOAD_MAX_MEMORY_SIZE |
| 246 |
): |
| 247 |
raise RequestDataTooBig( |
| 248 |
"Request body exceeded " |
| 249 |
"settings.DATA_UPLOAD_MAX_MEMORY_SIZE." |
| 250 |
) |
| 251 |
|
| 252 |
self._post.appendlist( |
| 253 |
field_name, force_str(data, encoding, errors="replace") |
| 254 |
) |
| 255 |
elif item_type == FILE: |
| 256 |
# Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FILES. |
| 257 |
num_files += 1 |
| 258 |
if ( |
| 259 |
settings.DATA_UPLOAD_MAX_NUMBER_FILES is not None |
| 260 |
and num_files > settings.DATA_UPLOAD_MAX_NUMBER_FILES |
| 261 |
): |
| 262 |
raise TooManyFilesSent( |
| 263 |
"The number of files exceeded " |
| 264 |
"settings.DATA_UPLOAD_MAX_NUMBER_FILES." |
| 265 |
) |
| 266 |
# This is a file, use the handler... |
| 267 |
file_name = disposition.get("filename") |
| 268 |
if file_name: |
| 269 |
file_name = force_str(file_name, encoding, errors="replace") |
| 270 |
file_name = self.sanitize_file_name(file_name) |
| 271 |
if not file_name: |
| 272 |
continue |
| 273 |
|
| 274 |
content_type, content_type_extra = meta_data.get( |
| 275 |
"content-type", ("", {}) |
| 276 |
) |
| 277 |
content_type = content_type.strip() |
| 278 |
charset = content_type_extra.get("charset") |
| 279 |
|
| 280 |
try: |
| 281 |
content_length = int(meta_data.get("content-length")[0]) |
| 282 |
except (IndexError, TypeError, ValueError): |
| 283 |
content_length = None |
| 284 |
|
| 285 |
counters = [0] * len(handlers) |
| 286 |
uploaded_file = False |
| 287 |
try: |
| 288 |
for handler in handlers: |
| 289 |
try: |
| 290 |
handler.new_file( |
| 291 |
field_name, |
| 292 |
file_name, |
| 293 |
content_type, |
| 294 |
content_length, |
| 295 |
charset, |
| 296 |
content_type_extra, |
| 297 |
) |
| 298 |
except StopFutureHandlers: |
| 299 |
break |
| 300 |
|
| 301 |
for chunk in field_stream: |
| 302 |
if transfer_encoding == "base64": |
| 303 |
# We only special-case base64 transfer encoding |
| 304 |
# We should always decode base64 chunks by |
| 305 |
# multiple of 4, ignoring whitespace. |
| 306 |
|
| 307 |
stripped_parts = [b"".join(chunk.split())] |
| 308 |
stripped_length = len(stripped_parts[0]) |
| 309 |
|
| 310 |
while stripped_length % 4 != 0: |
| 311 |
over_chunk = field_stream.read(self._chunk_size) |
| 312 |
if not over_chunk: |
| 313 |
break |
| 314 |
over_stripped = b"".join(over_chunk.split()) |
| 315 |
stripped_parts.append(over_stripped) |
| 316 |
stripped_length += len(over_stripped) |
| 317 |
|
| 318 |
stripped_chunk = b"".join(stripped_parts) |
| 319 |
|
| 320 |
try: |
| 321 |
chunk = base64.b64decode(stripped_chunk) |
| 322 |
except Exception as exc: |
| 323 |
# Since this is only a chunk, any error is |
| 324 |
# an unfixable error. |
| 325 |
raise MultiPartParserError( |
| 326 |
"Could not decode base64 data." |
| 327 |
) from exc |
| 328 |
|
| 329 |
for i, handler in enumerate(handlers): |
| 330 |
chunk_length = len(chunk) |
| 331 |
chunk = handler.receive_data_chunk(chunk, counters[i]) |
| 332 |
counters[i] += chunk_length |
| 333 |
if chunk is None: |
| 334 |
# Don't continue if the chunk received by |
| 335 |
# the handler is None. |
| 336 |
break |
| 337 |
|
| 338 |
except SkipFile: |
| 339 |
self._close_files() |
| 340 |
# Just use up the rest of this file... |
| 341 |
exhaust(field_stream) |
| 342 |
else: |
| 343 |
# Handle file upload completions on next iteration. |
| 344 |
old_field_name = field_name |
| 345 |
else: |
| 346 |
# If this is neither a FIELD nor a FILE, exhaust the field |
| 347 |
# stream. Note: There could be an error here at some point, |
| 348 |
# but there will be at least two RAW types (before and |
| 349 |
# after the other boundaries). This branch is usually not |
| 350 |
# reached at all, because a missing content-disposition |
| 351 |
# header will skip the whole boundary. |
| 352 |
exhaust(field_stream) |
| 353 |
except StopUpload as e: |
| 354 |
self._close_files() |
| 355 |
if not e.connection_reset: |
| 356 |
exhaust(self._input_data) |
| 357 |
else: |
| 358 |
if not uploaded_file: |
| 359 |
for handler in handlers: |
| 360 |
handler.upload_interrupted() |
| 361 |
# Make sure that the request data is all fed |
| 362 |
exhaust(self._input_data) |
| 363 |
|
| 364 |
# Signal that the upload has completed. |
| 365 |
# any() shortcircuits if a handler's upload_complete() returns a value. |
| 366 |
any(handler.upload_complete() for handler in handlers) |
| 367 |
self._post._mutable = False |
| 368 |
return self._post, self._files |
| 369 |
|
| 370 |
def handle_file_complete(self, old_field_name, counters): |
| 371 |
""" |
| 372 |
Handle all the signaling that takes place when a file is complete. |
| 373 |
""" |
| 374 |
for i, handler in enumerate(self._upload_handlers): |
| 375 |
file_obj = handler.file_complete(counters[i]) |
| 376 |
if file_obj: |
| 377 |
# If it returns a file object, then set the files dict. |
| 378 |
self._files.appendlist( |
| 379 |
force_str(old_field_name, self._encoding, errors="replace"), |
| 380 |
file_obj, |
| 381 |
) |
| 382 |
break |
| 383 |
|
| 384 |
def sanitize_file_name(self, file_name): |
| 385 |
""" |
| 386 |
Sanitize the filename of an upload. |
| 387 |
|
| 388 |
Remove all possible path separators, even though that might remove more |
| 389 |
than actually required by the target system. Filenames that could |
| 390 |
potentially cause problems (current/parent dir) are also discarded. |
| 391 |
|
| 392 |
It should be noted that this function could still return a "filepath" |
| 393 |
like "C:some_file.txt" which is handled later on by the storage layer. |
| 394 |
So while this function does sanitize filenames to some extent, the |
| 395 |
resulting filename should still be considered as untrusted user input. |
| 396 |
""" |
| 397 |
file_name = html.unescape(file_name) |
| 398 |
file_name = file_name.rsplit("/")[-1] |
| 399 |
file_name = file_name.rsplit("\\")[-1] |
| 400 |
# Remove non-printable characters. |
| 401 |
file_name = "".join([char for char in file_name if char.isprintable()]) |
| 402 |
|
| 403 |
if file_name in {"", ".", ".."}: |
| 404 |
return None |
| 405 |
return file_name |
| 406 |
|
| 407 |
IE_sanitize = sanitize_file_name |
| 408 |
|
| 409 |
def _close_files(self): |
| 410 |
# Free up all file handles. |
| 411 |
# FIXME: this currently assumes that upload handlers store the file as 'file' |
| 412 |
# We should document that... |
| 413 |
# (Maybe add handler.free_file to complement new_file) |
| 414 |
for handler in self._upload_handlers: |
| 415 |
if hasattr(handler, "file"): |
| 416 |
handler.file.close() |
| 417 |
|
| 418 |
|
| 419 |
class LazyStream: |
| 420 |
""" |
| 421 |
The LazyStream wrapper allows one to get and "unget" bytes from a stream. |
| 422 |
|
| 423 |
Given a producer object (an iterator that yields bytestrings), the |
| 424 |
LazyStream object will support iteration, reading, and keeping a "look-back" |
| 425 |
variable in case you need to "unget" some bytes. |
| 426 |
""" |
| 427 |
|
| 428 |
def __init__(self, producer, length=None): |
| 429 |
""" |
| 430 |
Every LazyStream must have a producer when instantiated. |
| 431 |
|
| 432 |
A producer is an iterable that returns a string each time it |
| 433 |
is called. |
| 434 |
""" |
| 435 |
self._producer = producer |
| 436 |
self._empty = False |
| 437 |
self._leftover = b"" |
| 438 |
self.length = length |
| 439 |
self.position = 0 |
| 440 |
self._remaining = length |
| 441 |
self._unget_history = [] |
| 442 |
|
| 443 |
def tell(self): |
| 444 |
return self.position |
| 445 |
|
| 446 |
def read(self, size=None): |
| 447 |
def parts(): |
| 448 |
remaining = self._remaining if size is None else size |
| 449 |
# do the whole thing in one shot if no limit was provided. |
| 450 |
if remaining is None: |
| 451 |
yield b"".join(self) |
| 452 |
return |
| 453 |
|
| 454 |
# otherwise do some bookkeeping to return exactly enough |
| 455 |
# of the stream and stashing any extra content we get from |
| 456 |
# the producer |
| 457 |
while remaining != 0: |
| 458 |
assert remaining > 0, "remaining bytes to read should never go negative" |
| 459 |
|
| 460 |
try: |
| 461 |
chunk = next(self) |
| 462 |
except StopIteration: |
| 463 |
return |
| 464 |
else: |
| 465 |
emitting = chunk[:remaining] |
| 466 |
self.unget(chunk[remaining:]) |
| 467 |
remaining -= len(emitting) |
| 468 |
yield emitting |
| 469 |
|
| 470 |
return b"".join(parts()) |
| 471 |
|
| 472 |
def __next__(self): |
| 473 |
""" |
| 474 |
Used when the exact number of bytes to read is unimportant. |
| 475 |
|
| 476 |
Return whatever chunk is conveniently returned from the iterator. |
| 477 |
Useful to avoid unnecessary bookkeeping if performance is an issue. |
| 478 |
""" |
| 479 |
if self._leftover: |
| 480 |
output = self._leftover |
| 481 |
self._leftover = b"" |
| 482 |
else: |
| 483 |
output = next(self._producer) |
| 484 |
self._unget_history = [] |
| 485 |
self.position += len(output) |
| 486 |
return output |
| 487 |
|
| 488 |
def close(self): |
| 489 |
""" |
| 490 |
Used to invalidate/disable this lazy stream. |
| 491 |
|
| 492 |
Replace the producer with an empty list. Any leftover bytes that have |
| 493 |
already been read will still be reported upon read() and/or next(). |
| 494 |
""" |
| 495 |
self._producer = [] |
| 496 |
|
| 497 |
def __iter__(self): |
| 498 |
return self |
| 499 |
|
| 500 |
def unget(self, bytes): |
| 501 |
""" |
| 502 |
Place bytes back onto the front of the lazy stream. |
| 503 |
|
| 504 |
Future calls to read() will return those bytes first. The |
| 505 |
stream position and thus tell() will be rewound. |
| 506 |
""" |
| 507 |
if not bytes: |
| 508 |
return |
| 509 |
self._update_unget_history(len(bytes)) |
| 510 |
self.position -= len(bytes) |
| 511 |
self._leftover = bytes + self._leftover |
| 512 |
|
| 513 |
def _update_unget_history(self, num_bytes): |
| 514 |
""" |
| 515 |
Update the unget history as a sanity check to see if we've pushed |
| 516 |
back the same number of bytes in one chunk. If we keep ungetting the |
| 517 |
same number of bytes many times (here, 50), we're mostly likely in an |
| 518 |
infinite loop of some sort. This is usually caused by a |
| 519 |
maliciously-malformed MIME request. |
| 520 |
""" |
| 521 |
self._unget_history = [num_bytes] + self._unget_history[:49] |
| 522 |
number_equal = len( |
| 523 |
[ |
| 524 |
current_number |
| 525 |
for current_number in self._unget_history |
| 526 |
if current_number == num_bytes |
| 527 |
] |
| 528 |
) |
| 529 |
|
| 530 |
if number_equal > 40: |
| 531 |
raise SuspiciousMultipartForm( |
| 532 |
"The multipart parser got stuck, which shouldn't happen with" |
| 533 |
" normal uploaded files. Check for malicious upload activity;" |
| 534 |
" if there is none, report this to the Django developers." |
| 535 |
) |
| 536 |
|
| 537 |
|
| 538 |
class ChunkIter: |
| 539 |
""" |
| 540 |
An iterable that will yield chunks of data. Given a file-like object as the |
| 541 |
constructor, yield chunks of read operations from that object. |
| 542 |
""" |
| 543 |
|
| 544 |
def __init__(self, flo, chunk_size=64 * 1024): |
| 545 |
self.flo = flo |
| 546 |
self.chunk_size = chunk_size |
| 547 |
|
| 548 |
def __next__(self): |
| 549 |
try: |
| 550 |
data = self.flo.read(self.chunk_size) |
| 551 |
except InputStreamExhausted: |
| 552 |
raise StopIteration() |
| 553 |
if data: |
| 554 |
return data |
| 555 |
else: |
| 556 |
raise StopIteration() |
| 557 |
|
| 558 |
def __iter__(self): |
| 559 |
return self |
| 560 |
|
| 561 |
|
| 562 |
class InterBoundaryIter: |
| 563 |
""" |
| 564 |
A Producer that will iterate over boundaries. |
| 565 |
""" |
| 566 |
|
| 567 |
def __init__(self, stream, boundary): |
| 568 |
self._stream = stream |
| 569 |
self._boundary = boundary |
| 570 |
|
| 571 |
def __iter__(self): |
| 572 |
return self |
| 573 |
|
| 574 |
def __next__(self): |
| 575 |
try: |
| 576 |
return LazyStream(BoundaryIter(self._stream, self._boundary)) |
| 577 |
except InputStreamExhausted: |
| 578 |
raise StopIteration() |
| 579 |
|
| 580 |
|
| 581 |
class BoundaryIter: |
| 582 |
""" |
| 583 |
A Producer that is sensitive to boundaries. |
| 584 |
|
| 585 |
Will happily yield bytes until a boundary is found. Will yield the bytes |
| 586 |
before the boundary, throw away the boundary bytes themselves, and push the |
| 587 |
post-boundary bytes back on the stream. |
| 588 |
|
| 589 |
The future calls to next() after locating the boundary will raise a |
| 590 |
StopIteration exception. |
| 591 |
""" |
| 592 |
|
| 593 |
def __init__(self, stream, boundary): |
| 594 |
self._stream = stream |
| 595 |
self._boundary = boundary |
| 596 |
self._done = False |
| 597 |
# rollback an additional six bytes because the format is like |
| 598 |
# this: CRLF<boundary>[--CRLF] |
| 599 |
self._rollback = len(boundary) + 6 |
| 600 |
|
| 601 |
# Try to use mx fast string search if available. Otherwise |
| 602 |
# use Python find. Wrap the latter for consistency. |
| 603 |
unused_char = self._stream.read(1) |
| 604 |
if not unused_char: |
| 605 |
raise InputStreamExhausted() |
| 606 |
self._stream.unget(unused_char) |
| 607 |
|
| 608 |
def __iter__(self): |
| 609 |
return self |
| 610 |
|
| 611 |
def __next__(self): |
| 612 |
if self._done: |
| 613 |
raise StopIteration() |
| 614 |
|
| 615 |
stream = self._stream |
| 616 |
rollback = self._rollback |
| 617 |
|
| 618 |
bytes_read = 0 |
| 619 |
chunks = [] |
| 620 |
for bytes in stream: |
| 621 |
bytes_read += len(bytes) |
| 622 |
chunks.append(bytes) |
| 623 |
if bytes_read > rollback: |
| 624 |
break |
| 625 |
if not bytes: |
| 626 |
break |
| 627 |
else: |
| 628 |
self._done = True |
| 629 |
|
| 630 |
if not chunks: |
| 631 |
raise StopIteration() |
| 632 |
|
| 633 |
chunk = b"".join(chunks) |
| 634 |
boundary = self._find_boundary(chunk) |
| 635 |
|
| 636 |
if boundary: |
| 637 |
end, next = boundary |
| 638 |
stream.unget(chunk[next:]) |
| 639 |
self._done = True |
| 640 |
return chunk[:end] |
| 641 |
else: |
| 642 |
# make sure we don't treat a partial boundary (and |
| 643 |
# its separators) as data |
| 644 |
if not chunk[:-rollback]: # and len(chunk) >= (len(self._boundary) + 6): |
| 645 |
# There's nothing left, we should just return and mark as done. |
| 646 |
self._done = True |
| 647 |
return chunk |
| 648 |
else: |
| 649 |
stream.unget(chunk[-rollback:]) |
| 650 |
return chunk[:-rollback] |
| 651 |
|
| 652 |
def _find_boundary(self, data): |
| 653 |
""" |
| 654 |
Find a multipart boundary in data. |
| 655 |
|
| 656 |
Should no boundary exist in the data, return None. Otherwise, return |
| 657 |
a tuple containing the indices of the following: |
| 658 |
* the end of current encapsulation |
| 659 |
* the start of the next encapsulation |
| 660 |
""" |
| 661 |
index = data.find(self._boundary) |
| 662 |
if index < 0: |
| 663 |
return None |
| 664 |
else: |
| 665 |
end = index |
| 666 |
next = index + len(self._boundary) |
| 667 |
# backup over CRLF |
| 668 |
last = max(0, end - 1) |
| 669 |
if data[last : last + 1] == b"\n": |
| 670 |
end -= 1 |
| 671 |
last = max(0, end - 1) |
| 672 |
if data[last : last + 1] == b"\r": |
| 673 |
end -= 1 |
| 674 |
return end, next |
| 675 |
|
| 676 |
|
| 677 |
def exhaust(stream_or_iterable): |
| 678 |
"""Exhaust an iterator or stream.""" |
| 679 |
try: |
| 680 |
iterator = iter(stream_or_iterable) |
| 681 |
except TypeError: |
| 682 |
iterator = ChunkIter(stream_or_iterable, 16384) |
| 683 |
collections.deque(iterator, maxlen=0) # consume iterator quickly. |
| 684 |
|
| 685 |
|
| 686 |
def parse_boundary_stream(stream, max_header_size): |
| 687 |
""" |
| 688 |
Parse one and exactly one stream that encapsulates a boundary. |
| 689 |
""" |
| 690 |
|
| 691 |
# Look for the end of headers and if not found extend the search to double |
| 692 |
# the size up to the MAX_TOTAL_HEADER_SIZE. |
| 693 |
headers_chunk_size = 1024 |
| 694 |
while True: |
| 695 |
if headers_chunk_size > max_header_size: |
| 696 |
raise MultiPartParserError("Request max total header size exceeded.") |
| 697 |
|
| 698 |
# Stream at beginning of header, look for end of header and parse it if |
| 699 |
# found. The header must fit within one chunk. |
| 700 |
chunk = stream.read(headers_chunk_size) |
| 701 |
# 'find' returns the top of these four bytes, so munch them later to |
| 702 |
# prevent them from polluting the payload. |
| 703 |
header_end = chunk.find(b"\r\n\r\n") |
| 704 |
if header_end != -1: |
| 705 |
break |
| 706 |
|
| 707 |
# Find no header, mark this fact and pass on the stream verbatim. |
| 708 |
stream.unget(chunk) |
| 709 |
# No more data to read. |
| 710 |
if len(chunk) < headers_chunk_size: |
| 711 |
return (RAW, {}, stream) |
| 712 |
# Double the chunk size. |
| 713 |
headers_chunk_size *= 2 |
| 714 |
|
| 715 |
header = chunk[:header_end] |
| 716 |
|
| 717 |
# here we place any excess chunk back onto the stream, as |
| 718 |
# well as throwing away the CRLFCRLF bytes from above. |
| 719 |
stream.unget(chunk[header_end + 4 :]) |
| 720 |
|
| 721 |
TYPE = RAW |
| 722 |
outdict = {} |
| 723 |
|
| 724 |
# Eliminate blank lines |
| 725 |
for line in header.split(b"\r\n"): |
| 726 |
# This terminology ("main value" and "dictionary of |
| 727 |
# parameters") is from the Python docs. |
| 728 |
try: |
| 729 |
main_value_pair, params = parse_header_parameters(line.decode()) |
| 730 |
name, value = main_value_pair.split(":", 1) |
| 731 |
params = {k: v.encode() for k, v in params.items()} |
| 732 |
except ValueError: # Invalid header. |
| 733 |
continue |
| 734 |
|
| 735 |
if name == "content-disposition": |
| 736 |
TYPE = FIELD |
| 737 |
if params.get("filename"): |
| 738 |
TYPE = FILE |
| 739 |
|
| 740 |
outdict[name] = value, params |
| 741 |
|
| 742 |
if TYPE == RAW: |
| 743 |
stream.unget(chunk) |
| 744 |
|
| 745 |
return (TYPE, outdict, stream) |
| 746 |
|
| 747 |
|
| 748 |
class Parser: |
| 749 |
def __init__(self, stream, boundary): |
| 750 |
self._stream = stream |
| 751 |
self._separator = b"--" + boundary |
| 752 |
|
| 753 |
def __iter__(self): |
| 754 |
boundarystream = InterBoundaryIter(self._stream, self._separator) |
| 755 |
for sub_stream in boundarystream: |
| 756 |
# Iterate over each part |
| 757 |
yield parse_boundary_stream(sub_stream, MAX_TOTAL_HEADER_SIZE) |