Debian Codegraph @

django/http/multipartparser.py

raw
1 """
2 Multi-part parsing for file uploads.
3
4 Exposes one class, ``MultiPartParser``, which feeds chunks of uploaded data to
5 file upload handlers for processing.
6 """
7
8 import base64
9 import binascii
10 import collections
11 import html
12
13 from django.conf import settings
14 from django.core.exceptions import (
15 RequestDataTooBig,
16 SuspiciousMultipartForm,
17 TooManyFieldsSent,
18 TooManyFilesSent,
19 )
20 from django.core.files.uploadhandler import SkipFile, StopFutureHandlers, StopUpload
21 from django.utils.datastructures import MultiValueDict
22 from django.utils.encoding import force_str
23 from django.utils.http import parse_header_parameters
24 from django.utils.regex_helper import _lazy_re_compile
25
26 __all__ = ("MultiPartParser", "MultiPartParserError", "InputStreamExhausted")
27
28
29 class MultiPartParserError(Exception):
30 pass
31
32
33 class InputStreamExhausted(Exception):
34 """
35 No more reads are allowed from this device.
36 """
37
38 pass
39
40
41 RAW = "raw"
42 FILE = "file"
43 FIELD = "field"
44 FIELD_TYPES = frozenset([FIELD, RAW])
45 MAX_TOTAL_HEADER_SIZE = 1024
46
47
48 class MultiPartParser:
49 """
50 An RFC 7578 multipart/form-data parser.
51
52 ``MultiValueDict.parse()`` reads the input stream in ``chunk_size`` chunks
53 and returns a tuple of ``(MultiValueDict(POST), MultiValueDict(FILES))``.
54 """
55
56 boundary_re = _lazy_re_compile(r"[ -~]{0,200}[!-~]")
57
58 def __init__(self, META, input_data, upload_handlers, encoding=None):
59 """
60 Initialize the MultiPartParser object.
61
62 :META:
63 The standard ``META`` dictionary in Django request objects.
64 :input_data:
65 The raw post data, as a file-like object.
66 :upload_handlers:
67 A list of UploadHandler instances that perform operations on the
68 uploaded data.
69 :encoding:
70 The encoding with which to treat the incoming data.
71 """
72 # Content-Type should contain multipart and the boundary information.
73 content_type = META.get("CONTENT_TYPE", "")
74 if not content_type.startswith("multipart/"):
75 raise MultiPartParserError("Invalid Content-Type: %s" % content_type)
76
77 try:
78 content_type.encode("ascii")
79 except UnicodeEncodeError:
80 raise MultiPartParserError(
81 "Invalid non-ASCII Content-Type in multipart: %s"
82 % force_str(content_type)
83 )
84
85 # Parse the header to get the boundary to split the parts.
86 _, opts = parse_header_parameters(content_type)
87 boundary = opts.get("boundary")
88 if not boundary or not self.boundary_re.fullmatch(boundary):
89 raise MultiPartParserError(
90 "Invalid boundary in multipart: %s" % force_str(boundary)
91 )
92
93 # Content-Length should contain the length of the body we are about
94 # to receive.
95 try:
96 content_length = int(META.get("CONTENT_LENGTH", 0))
97 except (ValueError, TypeError):
98 content_length = 0
99
100 if content_length < 0:
101 # This means we shouldn't continue...raise an error.
102 raise MultiPartParserError("Invalid content length: %r" % content_length)
103
104 self._boundary = boundary.encode("ascii")
105 self._input_data = input_data
106
107 # For compatibility with low-level network APIs (with 32-bit integers),
108 # the chunk size should be < 2^31, but still divisible by 4.
109 possible_sizes = [x.chunk_size for x in upload_handlers if x.chunk_size]
110 self._chunk_size = min([2**31 - 4] + possible_sizes)
111
112 self._meta = META
113 self._encoding = encoding or settings.DEFAULT_CHARSET
114 self._content_length = content_length
115 self._upload_handlers = upload_handlers
116
117 def parse(self):
118 # Call the actual parse routine and close all open files in case of
119 # errors. This is needed because if exceptions are thrown the
120 # MultiPartParser will not be garbage collected immediately and
121 # resources would be kept alive. This is only needed for errors because
122 # the Request object closes all uploaded files at the end of the
123 # request.
124 try:
125 return self._parse()
126 except Exception:
127 if hasattr(self, "_files"):
128 for _, files in self._files.lists():
129 for fileobj in files:
130 fileobj.close()
131 raise
132
133 def _parse(self):
134 """
135 Parse the POST data and break it into a FILES MultiValueDict and a POST
136 MultiValueDict.
137
138 Return a tuple containing the POST and FILES dictionary, respectively.
139 """
140 from django.http import QueryDict
141
142 encoding = self._encoding
143 handlers = self._upload_handlers
144
145 # HTTP spec says that Content-Length >= 0 is valid
146 # handling content-length == 0 before continuing
147 if self._content_length == 0:
148 return QueryDict(encoding=self._encoding), MultiValueDict()
149
150 # See if any of the handlers take care of the parsing.
151 # This allows overriding everything if need be.
152 for handler in handlers:
153 result = handler.handle_raw_input(
154 self._input_data,
155 self._meta,
156 self._content_length,
157 self._boundary,
158 encoding,
159 )
160 # Check to see if it was handled
161 if result is not None:
162 return result[0], result[1]
163
164 # Create the data structures to be used later.
165 self._post = QueryDict(mutable=True)
166 self._files = MultiValueDict()
167
168 # Instantiate the parser and stream:
169 stream = LazyStream(ChunkIter(self._input_data, self._chunk_size))
170
171 # Whether or not to signal a file-completion at the beginning of the loop.
172 old_field_name = None
173 counters = [0] * len(handlers)
174
175 # Number of bytes that have been read.
176 num_bytes_read = 0
177 # To count the number of keys in the request.
178 num_post_keys = 0
179 # To count the number of files in the request.
180 num_files = 0
181 # To limit the amount of data read from the request.
182 read_size = None
183 # Whether a file upload is finished.
184 uploaded_file = True
185
186 try:
187 for item_type, meta_data, field_stream in Parser(stream, self._boundary):
188 if old_field_name:
189 # We run this at the beginning of the next loop
190 # since we cannot be sure a file is complete until
191 # we hit the next boundary/part of the multipart content.
192 self.handle_file_complete(old_field_name, counters)
193 old_field_name = None
194 uploaded_file = True
195
196 if (
197 item_type in FIELD_TYPES
198 and settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None
199 ):
200 # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS.
201 num_post_keys += 1
202 # 2 accounts for empty raw fields before and after the
203 # last boundary.
204 if settings.DATA_UPLOAD_MAX_NUMBER_FIELDS + 2 < num_post_keys:
205 raise TooManyFieldsSent(
206 "The number of GET/POST parameters exceeded "
207 "settings.DATA_UPLOAD_MAX_NUMBER_FIELDS."
208 )
209
210 try:
211 disposition = meta_data["content-disposition"][1]
212 field_name = disposition["name"].strip()
213 except (KeyError, IndexError, AttributeError):
214 continue
215
216 transfer_encoding = meta_data.get("content-transfer-encoding")
217 if transfer_encoding is not None:
218 transfer_encoding = transfer_encoding[0].strip()
219 field_name = force_str(field_name, encoding, errors="replace")
220
221 if item_type == FIELD:
222 # Avoid reading more than DATA_UPLOAD_MAX_MEMORY_SIZE.
223 if settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None:
224 read_size = (
225 settings.DATA_UPLOAD_MAX_MEMORY_SIZE - num_bytes_read
226 )
227
228 # This is a post field, we can just set it in the post
229 if transfer_encoding == "base64":
230 raw_data = field_stream.read(size=read_size)
231 num_bytes_read += len(raw_data)
232 try:
233 data = base64.b64decode(raw_data)
234 except binascii.Error:
235 data = raw_data
236 else:
237 data = field_stream.read(size=read_size)
238 num_bytes_read += len(data)
239
240 # Add two here to make the check consistent with the
241 # x-www-form-urlencoded check that includes '&='.
242 num_bytes_read += len(field_name) + 2
243 if (
244 settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None
245 and num_bytes_read > settings.DATA_UPLOAD_MAX_MEMORY_SIZE
246 ):
247 raise RequestDataTooBig(
248 "Request body exceeded "
249 "settings.DATA_UPLOAD_MAX_MEMORY_SIZE."
250 )
251
252 self._post.appendlist(
253 field_name, force_str(data, encoding, errors="replace")
254 )
255 elif item_type == FILE:
256 # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FILES.
257 num_files += 1
258 if (
259 settings.DATA_UPLOAD_MAX_NUMBER_FILES is not None
260 and num_files > settings.DATA_UPLOAD_MAX_NUMBER_FILES
261 ):
262 raise TooManyFilesSent(
263 "The number of files exceeded "
264 "settings.DATA_UPLOAD_MAX_NUMBER_FILES."
265 )
266 # This is a file, use the handler...
267 file_name = disposition.get("filename")
268 if file_name:
269 file_name = force_str(file_name, encoding, errors="replace")
270 file_name = self.sanitize_file_name(file_name)
271 if not file_name:
272 continue
273
274 content_type, content_type_extra = meta_data.get(
275 "content-type", ("", {})
276 )
277 content_type = content_type.strip()
278 charset = content_type_extra.get("charset")
279
280 try:
281 content_length = int(meta_data.get("content-length")[0])
282 except (IndexError, TypeError, ValueError):
283 content_length = None
284
285 counters = [0] * len(handlers)
286 uploaded_file = False
287 try:
288 for handler in handlers:
289 try:
290 handler.new_file(
291 field_name,
292 file_name,
293 content_type,
294 content_length,
295 charset,
296 content_type_extra,
297 )
298 except StopFutureHandlers:
299 break
300
301 for chunk in field_stream:
302 if transfer_encoding == "base64":
303 # We only special-case base64 transfer encoding
304 # We should always decode base64 chunks by
305 # multiple of 4, ignoring whitespace.
306
307 stripped_parts = [b"".join(chunk.split())]
308 stripped_length = len(stripped_parts[0])
309
310 while stripped_length % 4 != 0:
311 over_chunk = field_stream.read(self._chunk_size)
312 if not over_chunk:
313 break
314 over_stripped = b"".join(over_chunk.split())
315 stripped_parts.append(over_stripped)
316 stripped_length += len(over_stripped)
317
318 stripped_chunk = b"".join(stripped_parts)
319
320 try:
321 chunk = base64.b64decode(stripped_chunk)
322 except Exception as exc:
323 # Since this is only a chunk, any error is
324 # an unfixable error.
325 raise MultiPartParserError(
326 "Could not decode base64 data."
327 ) from exc
328
329 for i, handler in enumerate(handlers):
330 chunk_length = len(chunk)
331 chunk = handler.receive_data_chunk(chunk, counters[i])
332 counters[i] += chunk_length
333 if chunk is None:
334 # Don't continue if the chunk received by
335 # the handler is None.
336 break
337
338 except SkipFile:
339 self._close_files()
340 # Just use up the rest of this file...
341 exhaust(field_stream)
342 else:
343 # Handle file upload completions on next iteration.
344 old_field_name = field_name
345 else:
346 # If this is neither a FIELD nor a FILE, exhaust the field
347 # stream. Note: There could be an error here at some point,
348 # but there will be at least two RAW types (before and
349 # after the other boundaries). This branch is usually not
350 # reached at all, because a missing content-disposition
351 # header will skip the whole boundary.
352 exhaust(field_stream)
353 except StopUpload as e:
354 self._close_files()
355 if not e.connection_reset:
356 exhaust(self._input_data)
357 else:
358 if not uploaded_file:
359 for handler in handlers:
360 handler.upload_interrupted()
361 # Make sure that the request data is all fed
362 exhaust(self._input_data)
363
364 # Signal that the upload has completed.
365 # any() shortcircuits if a handler's upload_complete() returns a value.
366 any(handler.upload_complete() for handler in handlers)
367 self._post._mutable = False
368 return self._post, self._files
369
370 def handle_file_complete(self, old_field_name, counters):
371 """
372 Handle all the signaling that takes place when a file is complete.
373 """
374 for i, handler in enumerate(self._upload_handlers):
375 file_obj = handler.file_complete(counters[i])
376 if file_obj:
377 # If it returns a file object, then set the files dict.
378 self._files.appendlist(
379 force_str(old_field_name, self._encoding, errors="replace"),
380 file_obj,
381 )
382 break
383
384 def sanitize_file_name(self, file_name):
385 """
386 Sanitize the filename of an upload.
387
388 Remove all possible path separators, even though that might remove more
389 than actually required by the target system. Filenames that could
390 potentially cause problems (current/parent dir) are also discarded.
391
392 It should be noted that this function could still return a "filepath"
393 like "C:some_file.txt" which is handled later on by the storage layer.
394 So while this function does sanitize filenames to some extent, the
395 resulting filename should still be considered as untrusted user input.
396 """
397 file_name = html.unescape(file_name)
398 file_name = file_name.rsplit("/")[-1]
399 file_name = file_name.rsplit("\\")[-1]
400 # Remove non-printable characters.
401 file_name = "".join([char for char in file_name if char.isprintable()])
402
403 if file_name in {"", ".", ".."}:
404 return None
405 return file_name
406
407 IE_sanitize = sanitize_file_name
408
409 def _close_files(self):
410 # Free up all file handles.
411 # FIXME: this currently assumes that upload handlers store the file as 'file'
412 # We should document that...
413 # (Maybe add handler.free_file to complement new_file)
414 for handler in self._upload_handlers:
415 if hasattr(handler, "file"):
416 handler.file.close()
417
418
419 class LazyStream:
420 """
421 The LazyStream wrapper allows one to get and "unget" bytes from a stream.
422
423 Given a producer object (an iterator that yields bytestrings), the
424 LazyStream object will support iteration, reading, and keeping a "look-back"
425 variable in case you need to "unget" some bytes.
426 """
427
428 def __init__(self, producer, length=None):
429 """
430 Every LazyStream must have a producer when instantiated.
431
432 A producer is an iterable that returns a string each time it
433 is called.
434 """
435 self._producer = producer
436 self._empty = False
437 self._leftover = b""
438 self.length = length
439 self.position = 0
440 self._remaining = length
441 self._unget_history = []
442
443 def tell(self):
444 return self.position
445
446 def read(self, size=None):
447 def parts():
448 remaining = self._remaining if size is None else size
449 # do the whole thing in one shot if no limit was provided.
450 if remaining is None:
451 yield b"".join(self)
452 return
453
454 # otherwise do some bookkeeping to return exactly enough
455 # of the stream and stashing any extra content we get from
456 # the producer
457 while remaining != 0:
458 assert remaining > 0, "remaining bytes to read should never go negative"
459
460 try:
461 chunk = next(self)
462 except StopIteration:
463 return
464 else:
465 emitting = chunk[:remaining]
466 self.unget(chunk[remaining:])
467 remaining -= len(emitting)
468 yield emitting
469
470 return b"".join(parts())
471
472 def __next__(self):
473 """
474 Used when the exact number of bytes to read is unimportant.
475
476 Return whatever chunk is conveniently returned from the iterator.
477 Useful to avoid unnecessary bookkeeping if performance is an issue.
478 """
479 if self._leftover:
480 output = self._leftover
481 self._leftover = b""
482 else:
483 output = next(self._producer)
484 self._unget_history = []
485 self.position += len(output)
486 return output
487
488 def close(self):
489 """
490 Used to invalidate/disable this lazy stream.
491
492 Replace the producer with an empty list. Any leftover bytes that have
493 already been read will still be reported upon read() and/or next().
494 """
495 self._producer = []
496
497 def __iter__(self):
498 return self
499
500 def unget(self, bytes):
501 """
502 Place bytes back onto the front of the lazy stream.
503
504 Future calls to read() will return those bytes first. The
505 stream position and thus tell() will be rewound.
506 """
507 if not bytes:
508 return
509 self._update_unget_history(len(bytes))
510 self.position -= len(bytes)
511 self._leftover = bytes + self._leftover
512
513 def _update_unget_history(self, num_bytes):
514 """
515 Update the unget history as a sanity check to see if we've pushed
516 back the same number of bytes in one chunk. If we keep ungetting the
517 same number of bytes many times (here, 50), we're mostly likely in an
518 infinite loop of some sort. This is usually caused by a
519 maliciously-malformed MIME request.
520 """
521 self._unget_history = [num_bytes] + self._unget_history[:49]
522 number_equal = len(
523 [
524 current_number
525 for current_number in self._unget_history
526 if current_number == num_bytes
527 ]
528 )
529
530 if number_equal > 40:
531 raise SuspiciousMultipartForm(
532 "The multipart parser got stuck, which shouldn't happen with"
533 " normal uploaded files. Check for malicious upload activity;"
534 " if there is none, report this to the Django developers."
535 )
536
537
538 class ChunkIter:
539 """
540 An iterable that will yield chunks of data. Given a file-like object as the
541 constructor, yield chunks of read operations from that object.
542 """
543
544 def __init__(self, flo, chunk_size=64 * 1024):
545 self.flo = flo
546 self.chunk_size = chunk_size
547
548 def __next__(self):
549 try:
550 data = self.flo.read(self.chunk_size)
551 except InputStreamExhausted:
552 raise StopIteration()
553 if data:
554 return data
555 else:
556 raise StopIteration()
557
558 def __iter__(self):
559 return self
560
561
562 class InterBoundaryIter:
563 """
564 A Producer that will iterate over boundaries.
565 """
566
567 def __init__(self, stream, boundary):
568 self._stream = stream
569 self._boundary = boundary
570
571 def __iter__(self):
572 return self
573
574 def __next__(self):
575 try:
576 return LazyStream(BoundaryIter(self._stream, self._boundary))
577 except InputStreamExhausted:
578 raise StopIteration()
579
580
581 class BoundaryIter:
582 """
583 A Producer that is sensitive to boundaries.
584
585 Will happily yield bytes until a boundary is found. Will yield the bytes
586 before the boundary, throw away the boundary bytes themselves, and push the
587 post-boundary bytes back on the stream.
588
589 The future calls to next() after locating the boundary will raise a
590 StopIteration exception.
591 """
592
593 def __init__(self, stream, boundary):
594 self._stream = stream
595 self._boundary = boundary
596 self._done = False
597 # rollback an additional six bytes because the format is like
598 # this: CRLF<boundary>[--CRLF]
599 self._rollback = len(boundary) + 6
600
601 # Try to use mx fast string search if available. Otherwise
602 # use Python find. Wrap the latter for consistency.
603 unused_char = self._stream.read(1)
604 if not unused_char:
605 raise InputStreamExhausted()
606 self._stream.unget(unused_char)
607
608 def __iter__(self):
609 return self
610
611 def __next__(self):
612 if self._done:
613 raise StopIteration()
614
615 stream = self._stream
616 rollback = self._rollback
617
618 bytes_read = 0
619 chunks = []
620 for bytes in stream:
621 bytes_read += len(bytes)
622 chunks.append(bytes)
623 if bytes_read > rollback:
624 break
625 if not bytes:
626 break
627 else:
628 self._done = True
629
630 if not chunks:
631 raise StopIteration()
632
633 chunk = b"".join(chunks)
634 boundary = self._find_boundary(chunk)
635
636 if boundary:
637 end, next = boundary
638 stream.unget(chunk[next:])
639 self._done = True
640 return chunk[:end]
641 else:
642 # make sure we don't treat a partial boundary (and
643 # its separators) as data
644 if not chunk[:-rollback]: # and len(chunk) >= (len(self._boundary) + 6):
645 # There's nothing left, we should just return and mark as done.
646 self._done = True
647 return chunk
648 else:
649 stream.unget(chunk[-rollback:])
650 return chunk[:-rollback]
651
652 def _find_boundary(self, data):
653 """
654 Find a multipart boundary in data.
655
656 Should no boundary exist in the data, return None. Otherwise, return
657 a tuple containing the indices of the following:
658 * the end of current encapsulation
659 * the start of the next encapsulation
660 """
661 index = data.find(self._boundary)
662 if index < 0:
663 return None
664 else:
665 end = index
666 next = index + len(self._boundary)
667 # backup over CRLF
668 last = max(0, end - 1)
669 if data[last : last + 1] == b"\n":
670 end -= 1
671 last = max(0, end - 1)
672 if data[last : last + 1] == b"\r":
673 end -= 1
674 return end, next
675
676
677 def exhaust(stream_or_iterable):
678 """Exhaust an iterator or stream."""
679 try:
680 iterator = iter(stream_or_iterable)
681 except TypeError:
682 iterator = ChunkIter(stream_or_iterable, 16384)
683 collections.deque(iterator, maxlen=0) # consume iterator quickly.
684
685
686 def parse_boundary_stream(stream, max_header_size):
687 """
688 Parse one and exactly one stream that encapsulates a boundary.
689 """
690
691 # Look for the end of headers and if not found extend the search to double
692 # the size up to the MAX_TOTAL_HEADER_SIZE.
693 headers_chunk_size = 1024
694 while True:
695 if headers_chunk_size > max_header_size:
696 raise MultiPartParserError("Request max total header size exceeded.")
697
698 # Stream at beginning of header, look for end of header and parse it if
699 # found. The header must fit within one chunk.
700 chunk = stream.read(headers_chunk_size)
701 # 'find' returns the top of these four bytes, so munch them later to
702 # prevent them from polluting the payload.
703 header_end = chunk.find(b"\r\n\r\n")
704 if header_end != -1:
705 break
706
707 # Find no header, mark this fact and pass on the stream verbatim.
708 stream.unget(chunk)
709 # No more data to read.
710 if len(chunk) < headers_chunk_size:
711 return (RAW, {}, stream)
712 # Double the chunk size.
713 headers_chunk_size *= 2
714
715 header = chunk[:header_end]
716
717 # here we place any excess chunk back onto the stream, as
718 # well as throwing away the CRLFCRLF bytes from above.
719 stream.unget(chunk[header_end + 4 :])
720
721 TYPE = RAW
722 outdict = {}
723
724 # Eliminate blank lines
725 for line in header.split(b"\r\n"):
726 # This terminology ("main value" and "dictionary of
727 # parameters") is from the Python docs.
728 try:
729 main_value_pair, params = parse_header_parameters(line.decode())
730 name, value = main_value_pair.split(":", 1)
731 params = {k: v.encode() for k, v in params.items()}
732 except ValueError: # Invalid header.
733 continue
734
735 if name == "content-disposition":
736 TYPE = FIELD
737 if params.get("filename"):
738 TYPE = FILE
739
740 outdict[name] = value, params
741
742 if TYPE == RAW:
743 stream.unget(chunk)
744
745 return (TYPE, outdict, stream)
746
747
748 class Parser:
749 def __init__(self, stream, boundary):
750 self._stream = stream
751 self._separator = b"--" + boundary
752
753 def __iter__(self):
754 boundarystream = InterBoundaryIter(self._stream, self._separator)
755 for sub_stream in boundarystream:
756 # Iterate over each part
757 yield parse_boundary_stream(sub_stream, MAX_TOTAL_HEADER_SIZE)