Debian Codegraph @

Lib/http/cookiejar.py

raw
1 r"""HTTP cookie handling for web clients.
2
3 This module has (now fairly distant) origins in Gisle Aas' Perl module
4 HTTP::Cookies, from the libwww-perl library.
5
6 Docstrings, comments and debug strings in this code refer to the
7 attributes of the HTTP cookie system as cookie-attributes, to distinguish
8 them clearly from Python attributes.
9
10 Class diagram (note that BSDDBCookieJar and the MSIE* classes are not
11 distributed with the Python standard library, but are available from
12 http://wwwsearch.sf.net/):
13
14 CookieJar____
15 / \ \
16 FileCookieJar \ \
17 / | \ \ \
18 MozillaCookieJar | LWPCookieJar \ \
19 | | \
20 | ---MSIEBase | \
21 | / | | \
22 | / MSIEDBCookieJar BSDDBCookieJar
23 |/
24 MSIECookieJar
25
26 """
27
28 __all__ = ['Cookie', 'CookieJar', 'CookiePolicy', 'DefaultCookiePolicy',
29 'FileCookieJar', 'LWPCookieJar', 'LoadError', 'MozillaCookieJar']
30
31 import os
32 import copy
33 import datetime
34 import re
35 import time
36 import urllib.parse, urllib.request
37 import threading as _threading
38 import http.client # only for the default HTTP port
39 from calendar import timegm
40
41 debug = False # set to True to enable debugging via the logging module
42 logger = None
43
44 def _debug(*args):
45 if not debug:
46 return
47 global logger
48 if not logger:
49 import logging
50 logger = logging.getLogger("http.cookiejar")
51 return logger.debug(*args)
52
53 HTTPONLY_ATTR = "HTTPOnly"
54 HTTPONLY_PREFIX = "#HttpOnly_"
55 DEFAULT_HTTP_PORT = str(http.client.HTTP_PORT)
56 NETSCAPE_MAGIC_RGX = re.compile("#( Netscape)? HTTP Cookie File")
57 MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar "
58 "instance initialised with one)")
59 NETSCAPE_HEADER_TEXT = """\
60 # Netscape HTTP Cookie File
61 # http://curl.haxx.se/rfc/cookie_spec.html
62 # This is a generated file! Do not edit.
63
64 """
65
66 def _warn_unhandled_exception():
67 # There are a few catch-all except: statements in this module, for
68 # catching input that's bad in unexpected ways. Warn if any
69 # exceptions are caught there.
70 import io, warnings, traceback
71 f = io.StringIO()
72 traceback.print_exc(None, f)
73 msg = f.getvalue()
74 warnings.warn("http.cookiejar bug!\n%s" % msg, stacklevel=2)
75
76
77 # Date/time conversion
78 # -----------------------------------------------------------------------------
79
80 EPOCH_YEAR = 1970
81 def _timegm(tt):
82 year, month, mday, hour, min, sec = tt[:6]
83 if ((year >= EPOCH_YEAR) and (1 <= month <= 12) and (1 <= mday <= 31) and
84 (0 <= hour <= 24) and (0 <= min <= 59) and (0 <= sec <= 61)):
85 return timegm(tt)
86 else:
87 return None
88
89 DAYS = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
90 MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
91 "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
92 MONTHS_LOWER = [month.lower() for month in MONTHS]
93
94 def time2isoz(t=None):
95 """Return a string representing time in seconds since epoch, t.
96
97 If the function is called without an argument, it will use the current
98 time.
99
100 The format of the returned string is like "YYYY-MM-DD hh:mm:ssZ",
101 representing Universal Time (UTC, aka GMT). An example of this format is:
102
103 1994-11-24 08:49:37Z
104
105 """
106 if t is None:
107 dt = datetime.datetime.now(tz=datetime.UTC)
108 else:
109 dt = datetime.datetime.fromtimestamp(t, tz=datetime.UTC)
110 return "%04d-%02d-%02d %02d:%02d:%02dZ" % (
111 dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
112
113 def time2netscape(t=None):
114 """Return a string representing time in seconds since epoch, t.
115
116 If the function is called without an argument, it will use the current
117 time.
118
119 The format of the returned string is like this:
120
121 Wed, DD-Mon-YYYY HH:MM:SS GMT
122
123 """
124 if t is None:
125 dt = datetime.datetime.now(tz=datetime.UTC)
126 else:
127 dt = datetime.datetime.fromtimestamp(t, tz=datetime.UTC)
128 return "%s, %02d-%s-%04d %02d:%02d:%02d GMT" % (
129 DAYS[dt.weekday()], dt.day, MONTHS[dt.month-1],
130 dt.year, dt.hour, dt.minute, dt.second)
131
132
133 UTC_ZONES = {"GMT": None, "UTC": None, "UT": None, "Z": None}
134
135 TIMEZONE_RE = re.compile(r"^([-+])?(\d\d?):?(\d\d)?$", re.ASCII)
136 def offset_from_tz_string(tz):
137 offset = None
138 if tz in UTC_ZONES:
139 offset = 0
140 else:
141 m = TIMEZONE_RE.search(tz)
142 if m:
143 offset = 3600 * int(m.group(2))
144 if m.group(3):
145 offset = offset + 60 * int(m.group(3))
146 if m.group(1) == '-':
147 offset = -offset
148 return offset
149
150 def _str2time(day, mon, yr, hr, min, sec, tz):
151 yr = int(yr)
152 if yr > datetime.MAXYEAR:
153 return None
154
155 # translate month name to number
156 # month numbers start with 1 (January)
157 try:
158 mon = MONTHS_LOWER.index(mon.lower())+1
159 except ValueError:
160 # maybe it's already a number
161 try:
162 imon = int(mon)
163 except ValueError:
164 return None
165 if 1 <= imon <= 12:
166 mon = imon
167 else:
168 return None
169
170 # make sure clock elements are defined
171 if hr is None: hr = 0
172 if min is None: min = 0
173 if sec is None: sec = 0
174
175 day = int(day)
176 hr = int(hr)
177 min = int(min)
178 sec = int(sec)
179
180 if yr < 1000:
181 # find "obvious" year
182 cur_yr = time.localtime(time.time())[0]
183 m = cur_yr % 100
184 tmp = yr
185 yr = yr + cur_yr - m
186 m = m - tmp
187 if abs(m) > 50:
188 if m > 0: yr = yr + 100
189 else: yr = yr - 100
190
191 # convert UTC time tuple to seconds since epoch (not timezone-adjusted)
192 t = _timegm((yr, mon, day, hr, min, sec, tz))
193
194 if t is not None:
195 # adjust time using timezone string, to get absolute time since epoch
196 if tz is None:
197 tz = "UTC"
198 tz = tz.upper()
199 offset = offset_from_tz_string(tz)
200 if offset is None:
201 return None
202 t = t - offset
203
204 return t
205
206 STRICT_DATE_RE = re.compile(
207 r"^[SMTWF][a-z][a-z], (\d\d) ([JFMASOND][a-z][a-z]) "
208 r"(\d\d\d\d) (\d\d):(\d\d):(\d\d) GMT$", re.ASCII)
209 WEEKDAY_RE = re.compile(
210 r"^(?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)[a-z]*,?\s*", re.I | re.ASCII)
211 LOOSE_HTTP_DATE_RE = re.compile(
212 r"""^
213 (\d\d?) # day
214 (?:\s+|[-\/])
215 (\w+) # month
216 (?:\s+|[-\/])
217 (\d+) # year
218 (?:
219 (?:\s+|:) # separator before clock
220 (\d\d?):(\d\d) # hour:min
221 (?::(\d\d))? # optional seconds
222 )? # optional clock
223 \s*
224 (?:
225 ([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+) # timezone
226 \s*
227 )?
228 (?:
229 \(\w+\) # ASCII representation of timezone in parens.
230 \s*
231 )?$""", re.X | re.ASCII)
232 def http2time(text):
233 """Returns time in seconds since epoch of time represented by a string.
234
235 Return value is an integer.
236
237 None is returned if the format of str is unrecognized, the time is outside
238 the representable range, or the timezone string is not recognized. If the
239 string contains no timezone, UTC is assumed.
240
241 The timezone in the string may be numerical (like "-0800" or "+0100") or a
242 string timezone (like "UTC", "GMT", "BST" or "EST"). Currently, only the
243 timezone strings equivalent to UTC (zero offset) are known to the function.
244
245 The function loosely parses the following formats:
246
247 Wed, 09 Feb 1994 22:23:32 GMT -- HTTP format
248 Tuesday, 08-Feb-94 14:15:29 GMT -- old rfc850 HTTP format
249 Tuesday, 08-Feb-1994 14:15:29 GMT -- broken rfc850 HTTP format
250 09 Feb 1994 22:23:32 GMT -- HTTP format (no weekday)
251 08-Feb-94 14:15:29 GMT -- rfc850 format (no weekday)
252 08-Feb-1994 14:15:29 GMT -- broken rfc850 format (no weekday)
253
254 The parser ignores leading and trailing whitespace. The time may be
255 absent.
256
257 If the year is given with only 2 digits, the function will select the
258 century that makes the year closest to the current date.
259
260 """
261 # fast exit for strictly conforming string
262 m = STRICT_DATE_RE.search(text)
263 if m:
264 g = m.groups()
265 mon = MONTHS_LOWER.index(g[1].lower()) + 1
266 tt = (int(g[2]), mon, int(g[0]),
267 int(g[3]), int(g[4]), float(g[5]))
268 return _timegm(tt)
269
270 # No, we need some messy parsing...
271
272 # clean up
273 text = text.lstrip()
274 text = WEEKDAY_RE.sub("", text, 1) # Useless weekday
275
276 # tz is time zone specifier string
277 day, mon, yr, hr, min, sec, tz = [None]*7
278
279 # loose regexp parse
280 m = LOOSE_HTTP_DATE_RE.search(text)
281 if m is not None:
282 day, mon, yr, hr, min, sec, tz = m.groups()
283 else:
284 return None # bad format
285
286 return _str2time(day, mon, yr, hr, min, sec, tz)
287
288 ISO_DATE_RE = re.compile(
289 r"""^
290 (\d{4}) # year
291 [-\/]?
292 (\d\d?) # numerical month
293 [-\/]?
294 (\d\d?) # day
295 (?:
296 (?:\s+|[-:Tt]) # separator before clock
297 (\d\d?):?(\d\d) # hour:min
298 (?::?(\d\d(?:\.\d*)?))? # optional seconds (and fractional)
299 )? # optional clock
300 \s*
301 (?:
302 ([-+]?\d\d?:?(:?\d\d)?
303 |Z|z) # timezone (Z is "zero meridian", i.e. GMT)
304 \s*
305 )?$""", re.X | re. ASCII)
306 def iso2time(text):
307 """
308 As for http2time, but parses the ISO 8601 formats:
309
310 1994-02-03 14:15:29 -0100 -- ISO 8601 format
311 1994-02-03 14:15:29 -- zone is optional
312 1994-02-03 -- only date
313 1994-02-03T14:15:29 -- Use T as separator
314 19940203T141529Z -- ISO 8601 compact format
315 19940203 -- only date
316
317 """
318 # clean up
319 text = text.lstrip()
320
321 # tz is time zone specifier string
322 day, mon, yr, hr, min, sec, tz = [None]*7
323
324 # loose regexp parse
325 m = ISO_DATE_RE.search(text)
326 if m is not None:
327 # XXX there's an extra bit of the timezone I'm ignoring here: is
328 # this the right thing to do?
329 yr, mon, day, hr, min, sec, tz, _ = m.groups()
330 else:
331 return None # bad format
332
333 return _str2time(day, mon, yr, hr, min, sec, tz)
334
335
336 # Header parsing
337 # -----------------------------------------------------------------------------
338
339 def unmatched(match):
340 """Return unmatched part of re.Match object."""
341 start, end = match.span(0)
342 return match.string[:start]+match.string[end:]
343
344 HEADER_TOKEN_RE = re.compile(r"^\s*([^=\s;,]+)")
345 HEADER_QUOTED_VALUE_RE = re.compile(r"^\s*=\s*\"([^\"\\]*(?:\\.[^\"\\]*)*)\"")
346 HEADER_VALUE_RE = re.compile(r"^\s*=\s*([^\s;,]*)")
347 HEADER_ESCAPE_RE = re.compile(r"\\(.)")
348 def split_header_words(header_values):
349 r"""Parse header values into a list of lists containing key,value pairs.
350
351 The function knows how to deal with ",", ";" and "=" as well as quoted
352 values after "=". A list of space separated tokens are parsed as if they
353 were separated by ";".
354
355 If the header_values passed as argument contains multiple values, then they
356 are treated as if they were a single value separated by comma ",".
357
358 This means that this function is useful for parsing header fields that
359 follow this syntax (BNF as from the HTTP/1.1 specification, but we relax
360 the requirement for tokens).
361
362 headers = #header
363 header = (token | parameter) *( [";"] (token | parameter))
364
365 token = 1*<any CHAR except CTLs or separators>
366 separators = "(" | ")" | "<" | ">" | "@"
367 | "," | ";" | ":" | "\" | <">
368 | "/" | "[" | "]" | "?" | "="
369 | "{" | "}" | SP | HT
370
371 quoted-string = ( <"> *(qdtext | quoted-pair ) <"> )
372 qdtext = <any TEXT except <">>
373 quoted-pair = "\" CHAR
374
375 parameter = attribute "=" value
376 attribute = token
377 value = token | quoted-string
378
379 Each header is represented by a list of key/value pairs. The value for a
380 simple token (not part of a parameter) is None. Syntactically incorrect
381 headers will not necessarily be parsed as you would want.
382
383 This is easier to describe with some examples:
384
385 >>> split_header_words(['foo="bar"; port="80,81"; discard, bar=baz'])
386 [[('foo', 'bar'), ('port', '80,81'), ('discard', None)], [('bar', 'baz')]]
387 >>> split_header_words(['text/html; charset="iso-8859-1"'])
388 [[('text/html', None), ('charset', 'iso-8859-1')]]
389 >>> split_header_words([r'Basic realm="\"foo\bar\""'])
390 [[('Basic', None), ('realm', '"foobar"')]]
391
392 """
393 assert not isinstance(header_values, str)
394 result = []
395 for text in header_values:
396 orig_text = text
397 pairs = []
398 while text:
399 m = HEADER_TOKEN_RE.search(text)
400 if m:
401 text = unmatched(m)
402 name = m.group(1)
403 m = HEADER_QUOTED_VALUE_RE.search(text)
404 if m: # quoted value
405 text = unmatched(m)
406 value = m.group(1)
407 value = HEADER_ESCAPE_RE.sub(r"\1", value)
408 else:
409 m = HEADER_VALUE_RE.search(text)
410 if m: # unquoted value
411 text = unmatched(m)
412 value = m.group(1)
413 value = value.rstrip()
414 else:
415 # no value, a lone token
416 value = None
417 pairs.append((name, value))
418 elif text.lstrip().startswith(","):
419 # concatenated headers, as per RFC 2616 section 4.2
420 text = text.lstrip()[1:]
421 if pairs: result.append(pairs)
422 pairs = []
423 else:
424 # skip junk
425 non_junk, nr_junk_chars = re.subn(r"^[=\s;]*", "", text)
426 assert nr_junk_chars > 0, (
427 "split_header_words bug: '%s', '%s', %s" %
428 (orig_text, text, pairs))
429 text = non_junk
430 if pairs: result.append(pairs)
431 return result
432
433 HEADER_JOIN_TOKEN_RE = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
434 HEADER_JOIN_ESCAPE_RE = re.compile(r"([\"\\])")
435 def join_header_words(lists):
436 """Do the inverse (almost) of the conversion done by split_header_words.
437
438 Takes a list of lists of (key, value) pairs and produces a single header
439 value. Attribute values are quoted if needed.
440
441 >>> join_header_words([[("text/plain", None), ("charset", "iso-8859/1")]])
442 'text/plain; charset="iso-8859/1"'
443 >>> join_header_words([[("text/plain", None)], [("charset", "iso-8859/1")]])
444 'text/plain, charset="iso-8859/1"'
445
446 """
447 headers = []
448 for pairs in lists:
449 attr = []
450 for k, v in pairs:
451 if v is not None:
452 if not HEADER_JOIN_TOKEN_RE.fullmatch(v):
453 v = HEADER_JOIN_ESCAPE_RE.sub(r"\\\1", v) # escape " and \
454 v = '"%s"' % v
455 k = "%s=%s" % (k, v)
456 attr.append(k)
457 if attr: headers.append("; ".join(attr))
458 return ", ".join(headers)
459
460 def strip_quotes(text):
461 if text.startswith('"'):
462 text = text[1:]
463 if text.endswith('"'):
464 text = text[:-1]
465 return text
466
467 def parse_ns_headers(ns_headers):
468 """Ad-hoc parser for Netscape protocol cookie-attributes.
469
470 The old Netscape cookie format for Set-Cookie can for instance contain
471 an unquoted "," in the expires field, so we have to use this ad-hoc
472 parser instead of split_header_words.
473
474 XXX This may not make the best possible effort to parse all the crap
475 that Netscape Cookie headers contain. Ronald Tschalar's HTTPClient
476 parser is probably better, so could do worse than following that if
477 this ever gives any trouble.
478
479 Currently, this is also used for parsing RFC 2109 cookies.
480
481 """
482 known_attrs = ("expires", "domain", "path", "secure",
483 # RFC 2109 attrs (may turn up in Netscape cookies, too)
484 "version", "port", "max-age")
485
486 result = []
487 for ns_header in ns_headers:
488 pairs = []
489 version_set = False
490
491 # XXX: The following does not strictly adhere to RFCs in that empty
492 # names and values are legal (the former will only appear once and will
493 # be overwritten if multiple occurrences are present). This is
494 # mostly to deal with backwards compatibility.
495 for ii, param in enumerate(ns_header.split(';')):
496 param = param.strip()
497
498 key, sep, val = param.partition('=')
499 key = key.strip()
500
501 if not key:
502 if ii == 0:
503 break
504 else:
505 continue
506
507 # allow for a distinction between present and empty and missing
508 # altogether
509 val = val.strip() if sep else None
510
511 if ii != 0:
512 lc = key.lower()
513 if lc in known_attrs:
514 key = lc
515
516 if key == "version":
517 # This is an RFC 2109 cookie.
518 if val is not None:
519 val = strip_quotes(val)
520 version_set = True
521 elif key == "expires":
522 # convert expires date to seconds since epoch
523 if val is not None:
524 val = http2time(strip_quotes(val)) # None if invalid
525 pairs.append((key, val))
526
527 if pairs:
528 if not version_set:
529 pairs.append(("version", "0"))
530 result.append(pairs)
531
532 return result
533
534
535 IPV4_RE = re.compile(r"\.\d+$", re.ASCII)
536 def is_HDN(text):
537 """Return True if text is a host domain name."""
538 # XXX
539 # This may well be wrong. Which RFC is HDN defined in, if any (for
540 # the purposes of RFC 2965)?
541 # For the current implementation, what about IPv6? Remember to look
542 # at other uses of IPV4_RE also, if change this.
543 if IPV4_RE.search(text):
544 return False
545 if text == "":
546 return False
547 if text[0] == "." or text[-1] == ".":
548 return False
549 return True
550
551 def domain_match(A, B):
552 """Return True if domain A domain-matches domain B, according to RFC 2965.
553
554 A and B may be host domain names or IP addresses.
555
556 RFC 2965, section 1:
557
558 Host names can be specified either as an IP address or a HDN string.
559 Sometimes we compare one host name with another. (Such comparisons SHALL
560 be case-insensitive.) Host A's name domain-matches host B's if
561
562 * their host name strings string-compare equal; or
563
564 * A is a HDN string and has the form NB, where N is a non-empty
565 name string, B has the form .B', and B' is a HDN string. (So,
566 x.y.com domain-matches .Y.com but not Y.com.)
567
568 Note that domain-match is not a commutative operation: a.b.c.com
569 domain-matches .c.com, but not the reverse.
570
571 """
572 # Note that, if A or B are IP addresses, the only relevant part of the
573 # definition of the domain-match algorithm is the direct string-compare.
574 A = A.lower()
575 B = B.lower()
576 if A == B:
577 return True
578 if not is_HDN(A):
579 return False
580 i = A.rfind(B)
581 if i == -1 or i == 0:
582 # A does not have form NB, or N is the empty string
583 return False
584 if not B.startswith("."):
585 return False
586 if not is_HDN(B[1:]):
587 return False
588 return True
589
590 def liberal_is_HDN(text):
591 """Return True if text is a sort-of-like a host domain name.
592
593 For accepting/blocking domains.
594
595 """
596 if IPV4_RE.search(text):
597 return False
598 return True
599
600 def user_domain_match(A, B):
601 """For blocking/accepting domains.
602
603 A and B may be host domain names or IP addresses.
604
605 """
606 A = A.lower()
607 B = B.lower()
608 if not (liberal_is_HDN(A) and liberal_is_HDN(B)):
609 if A == B:
610 # equal IP addresses
611 return True
612 return False
613 initial_dot = B.startswith(".")
614 if initial_dot and A.endswith(B):
615 return True
616 if not initial_dot and A == B:
617 return True
618 return False
619
620 cut_port_re = re.compile(r":\d+$", re.ASCII)
621 def request_host(request):
622 """Return request-host, as defined by RFC 2965.
623
624 Variation from RFC: returned value is lowercased, for convenient
625 comparison.
626
627 """
628 url = request.get_full_url()
629 host = urllib.parse.urlparse(url)[1]
630 if host == "":
631 host = request.get_header("Host", "")
632
633 # remove port, if present
634 host = cut_port_re.sub("", host, 1)
635 return host.lower()
636
637 def eff_request_host(request):
638 """Return a tuple (request-host, effective request-host name).
639
640 As defined by RFC 2965, except both are lowercased.
641
642 """
643 erhn = req_host = request_host(request)
644 if "." not in req_host:
645 erhn = req_host + ".local"
646 return req_host, erhn
647
648 def request_path(request):
649 """Path component of request-URI, as defined by RFC 2965."""
650 url = request.get_full_url()
651 parts = urllib.parse.urlsplit(url)
652 path = escape_path(parts.path)
653 if not path.startswith("/"):
654 # fix bad RFC 2396 absoluteURI
655 path = "/" + path
656 return path
657
658 def request_port(request):
659 host = request.host
660 i = host.find(':')
661 if i >= 0:
662 port = host[i+1:]
663 try:
664 int(port)
665 except ValueError:
666 _debug("nonnumeric port: '%s'", port)
667 return None
668 else:
669 port = DEFAULT_HTTP_PORT
670 return port
671
672 # Characters in addition to A-Z, a-z, 0-9, '_', '.', and '-' that don't
673 # need to be escaped to form a valid HTTP URL (RFCs 2396 and 1738).
674 HTTP_PATH_SAFE = "%/;:@&=+$,!~*'()"
675 ESCAPED_CHAR_RE = re.compile(r"%([0-9a-fA-F][0-9a-fA-F])")
676 def uppercase_escaped_char(match):
677 return "%%%s" % match.group(1).upper()
678 def escape_path(path):
679 """Escape any invalid characters in HTTP URL, and uppercase all escapes."""
680 # There's no knowing what character encoding was used to create URLs
681 # containing %-escapes, but since we have to pick one to escape invalid
682 # path characters, we pick UTF-8, as recommended in the HTML 4.0
683 # specification:
684 # http://www.w3.org/TR/REC-html40/appendix/notes.html#h-B.2.1
685 # And here, kind of: draft-fielding-uri-rfc2396bis-03
686 # (And in draft IRI specification: draft-duerst-iri-05)
687 # (And here, for new URI schemes: RFC 2718)
688 path = urllib.parse.quote(path, HTTP_PATH_SAFE)
689 path = ESCAPED_CHAR_RE.sub(uppercase_escaped_char, path)
690 return path
691
692 def reach(h):
693 """Return reach of host h, as defined by RFC 2965, section 1.
694
695 The reach R of a host name H is defined as follows:
696
697 * If
698
699 - H is the host domain name of a host; and,
700
701 - H has the form A.B; and
702
703 - A has no embedded (that is, interior) dots; and
704
705 - B has at least one embedded dot, or B is the string "local".
706 then the reach of H is .B.
707
708 * Otherwise, the reach of H is H.
709
710 >>> reach("www.acme.com")
711 '.acme.com'
712 >>> reach("acme.com")
713 'acme.com'
714 >>> reach("acme.local")
715 '.local'
716
717 """
718 i = h.find(".")
719 if i >= 0:
720 #a = h[:i] # this line is only here to show what a is
721 b = h[i+1:]
722 i = b.find(".")
723 if is_HDN(h) and (i >= 0 or b == "local"):
724 return "."+b
725 return h
726
727 def is_third_party(request):
728 """
729
730 RFC 2965, section 3.3.6:
731
732 An unverifiable transaction is to a third-party host if its request-
733 host U does not domain-match the reach R of the request-host O in the
734 origin transaction.
735
736 """
737 req_host = request_host(request)
738 if not domain_match(req_host, reach(request.origin_req_host)):
739 return True
740 else:
741 return False
742
743
744 class Cookie:
745 """HTTP Cookie.
746
747 This class represents both Netscape and RFC 2965 cookies.
748
749 This is deliberately a very simple class. It just holds attributes. It's
750 possible to construct Cookie instances that don't comply with the cookie
751 standards. CookieJar.make_cookies is the factory function for Cookie
752 objects -- it deals with cookie parsing, supplying defaults, and
753 normalising to the representation used in this class. CookiePolicy is
754 responsible for checking them to see whether they should be accepted from
755 and returned to the server.
756
757 Note that the port may be present in the headers, but unspecified ("Port"
758 rather than"Port=80", for example); if this is the case, port is None.
759
760 """
761
762 def __init__(self, version, name, value,
763 port, port_specified,
764 domain, domain_specified, domain_initial_dot,
765 path, path_specified,
766 secure,
767 expires,
768 discard,
769 comment,
770 comment_url,
771 rest,
772 rfc2109=False,
773 ):
774
775 if version is not None: version = int(version)
776 if expires is not None: expires = int(float(expires))
777 if port is None and port_specified is True:
778 raise ValueError("if port is None, port_specified must be false")
779
780 self.version = version
781 self.name = name
782 self.value = value
783 self.port = port
784 self.port_specified = port_specified
785 # normalise case, as per RFC 2965 section 3.3.3
786 self.domain = domain.lower()
787 self.domain_specified = domain_specified
788 # Sigh. We need to know whether the domain given in the
789 # cookie-attribute had an initial dot, in order to follow RFC 2965
790 # (as clarified in draft errata). Needed for the returned $Domain
791 # value.
792 self.domain_initial_dot = domain_initial_dot
793 self.path = path
794 self.path_specified = path_specified
795 self.secure = secure
796 self.expires = expires
797 self.discard = discard
798 self.comment = comment
799 self.comment_url = comment_url
800 self.rfc2109 = rfc2109
801
802 self._rest = copy.copy(rest)
803
804 def has_nonstandard_attr(self, name):
805 return name in self._rest
806 def get_nonstandard_attr(self, name, default=None):
807 return self._rest.get(name, default)
808 def set_nonstandard_attr(self, name, value):
809 self._rest[name] = value
810
811 def is_expired(self, now=None):
812 if now is None: now = time.time()
813 if (self.expires is not None) and (self.expires <= now):
814 return True
815 return False
816
817 def __str__(self):
818 if self.port is None: p = ""
819 else: p = ":"+self.port
820 limit = self.domain + p + self.path
821 if self.value is not None:
822 namevalue = "%s=%s" % (self.name, self.value)
823 else:
824 namevalue = self.name
825 return "<Cookie %s for %s>" % (namevalue, limit)
826
827 def __repr__(self):
828 args = []
829 for name in ("version", "name", "value",
830 "port", "port_specified",
831 "domain", "domain_specified", "domain_initial_dot",
832 "path", "path_specified",
833 "secure", "expires", "discard", "comment", "comment_url",
834 ):
835 attr = getattr(self, name)
836 args.append("%s=%s" % (name, repr(attr)))
837 args.append("rest=%s" % repr(self._rest))
838 args.append("rfc2109=%s" % repr(self.rfc2109))
839 return "%s(%s)" % (self.__class__.__name__, ", ".join(args))
840
841
842 class CookiePolicy:
843 """Defines which cookies get accepted from and returned to server.
844
845 May also modify cookies, though this is probably a bad idea.
846
847 The subclass DefaultCookiePolicy defines the standard rules for Netscape
848 and RFC 2965 cookies -- override that if you want a customized policy.
849
850 """
851 def set_ok(self, cookie, request):
852 """Return true if (and only if) cookie should be accepted from server.
853
854 Currently, pre-expired cookies never get this far -- the CookieJar
855 class deletes such cookies itself.
856
857 """
858 raise NotImplementedError()
859
860 def return_ok(self, cookie, request):
861 """Return true if (and only if) cookie should be returned to server."""
862 raise NotImplementedError()
863
864 def domain_return_ok(self, domain, request):
865 """Return false if cookies should not be returned, given cookie domain.
866 """
867 return True
868
869 def path_return_ok(self, path, request):
870 """Return false if cookies should not be returned, given cookie path.
871 """
872 return True
873
874
875 class DefaultCookiePolicy(CookiePolicy):
876 """Implements the standard rules for accepting and returning cookies."""
877
878 DomainStrictNoDots = 1
879 DomainStrictNonDomain = 2
880 DomainRFC2965Match = 4
881
882 DomainLiberal = 0
883 DomainStrict = DomainStrictNoDots|DomainStrictNonDomain
884
885 def __init__(self,
886 blocked_domains=None, allowed_domains=None,
887 netscape=True, rfc2965=False,
888 rfc2109_as_netscape=None,
889 hide_cookie2=False,
890 strict_domain=False,
891 strict_rfc2965_unverifiable=True,
892 strict_ns_unverifiable=False,
893 strict_ns_domain=DomainLiberal,
894 strict_ns_set_initial_dollar=False,
895 strict_ns_set_path=False,
896 secure_protocols=("https", "wss")
897 ):
898 """Constructor arguments should be passed as keyword arguments only."""
899 self.netscape = netscape
900 self.rfc2965 = rfc2965
901 self.rfc2109_as_netscape = rfc2109_as_netscape
902 self.hide_cookie2 = hide_cookie2
903 self.strict_domain = strict_domain
904 self.strict_rfc2965_unverifiable = strict_rfc2965_unverifiable
905 self.strict_ns_unverifiable = strict_ns_unverifiable
906 self.strict_ns_domain = strict_ns_domain
907 self.strict_ns_set_initial_dollar = strict_ns_set_initial_dollar
908 self.strict_ns_set_path = strict_ns_set_path
909 self.secure_protocols = secure_protocols
910
911 if blocked_domains is not None:
912 self._blocked_domains = tuple(blocked_domains)
913 else:
914 self._blocked_domains = ()
915
916 if allowed_domains is not None:
917 allowed_domains = tuple(allowed_domains)
918 self._allowed_domains = allowed_domains
919
920 def blocked_domains(self):
921 """Return the sequence of blocked domains (as a tuple)."""
922 return self._blocked_domains
923 def set_blocked_domains(self, blocked_domains):
924 """Set the sequence of blocked domains."""
925 self._blocked_domains = tuple(blocked_domains)
926
927 def is_blocked(self, domain):
928 for blocked_domain in self._blocked_domains:
929 if user_domain_match(domain, blocked_domain):
930 return True
931 return False
932
933 def allowed_domains(self):
934 """Return None, or the sequence of allowed domains (as a tuple)."""
935 return self._allowed_domains
936 def set_allowed_domains(self, allowed_domains):
937 """Set the sequence of allowed domains, or None."""
938 if allowed_domains is not None:
939 allowed_domains = tuple(allowed_domains)
940 self._allowed_domains = allowed_domains
941
942 def is_not_allowed(self, domain):
943 if self._allowed_domains is None:
944 return False
945 for allowed_domain in self._allowed_domains:
946 if user_domain_match(domain, allowed_domain):
947 return False
948 return True
949
950 def set_ok(self, cookie, request):
951 """
952 If you override .set_ok(), be sure to call this method. If it returns
953 false, so should your subclass (assuming your subclass wants to be more
954 strict about which cookies to accept).
955
956 """
957 _debug(" - checking cookie %s=%s", cookie.name, cookie.value)
958
959 assert cookie.name is not None
960
961 for n in "version", "verifiability", "name", "path", "domain", "port":
962 fn_name = "set_ok_"+n
963 fn = getattr(self, fn_name)
964 if not fn(cookie, request):
965 return False
966
967 return True
968
969 def set_ok_version(self, cookie, request):
970 if cookie.version is None:
971 # Version is always set to 0 by parse_ns_headers if it's a Netscape
972 # cookie, so this must be an invalid RFC 2965 cookie.
973 _debug(" Set-Cookie2 without version attribute (%s=%s)",
974 cookie.name, cookie.value)
975 return False
976 if cookie.version > 0 and not self.rfc2965:
977 _debug(" RFC 2965 cookies are switched off")
978 return False
979 elif cookie.version == 0 and not self.netscape:
980 _debug(" Netscape cookies are switched off")
981 return False
982 return True
983
984 def set_ok_verifiability(self, cookie, request):
985 if request.unverifiable and is_third_party(request):
986 if cookie.version > 0 and self.strict_rfc2965_unverifiable:
987 _debug(" third-party RFC 2965 cookie during "
988 "unverifiable transaction")
989 return False
990 elif cookie.version == 0 and self.strict_ns_unverifiable:
991 _debug(" third-party Netscape cookie during "
992 "unverifiable transaction")
993 return False
994 return True
995
996 def set_ok_name(self, cookie, request):
997 # Try and stop servers setting V0 cookies designed to hack other
998 # servers that know both V0 and V1 protocols.
999 if (cookie.version == 0 and self.strict_ns_set_initial_dollar and
1000 cookie.name.startswith("$")):
1001 _debug(" illegal name (starts with '$'): '%s'", cookie.name)
1002 return False
1003 return True
1004
1005 def set_ok_path(self, cookie, request):
1006 if cookie.path_specified:
1007 req_path = request_path(request)
1008 if ((cookie.version > 0 or
1009 (cookie.version == 0 and self.strict_ns_set_path)) and
1010 not self.path_return_ok(cookie.path, request)):
1011 _debug(" path attribute %s is not a prefix of request "
1012 "path %s", cookie.path, req_path)
1013 return False
1014 return True
1015
1016 def set_ok_domain(self, cookie, request):
1017 if self.is_blocked(cookie.domain):
1018 _debug(" domain %s is in user block-list", cookie.domain)
1019 return False
1020 if self.is_not_allowed(cookie.domain):
1021 _debug(" domain %s is not in user allow-list", cookie.domain)
1022 return False
1023 if cookie.domain_specified:
1024 req_host, erhn = eff_request_host(request)
1025 domain = cookie.domain
1026 if self.strict_domain and (domain.count(".") >= 2):
1027 # XXX This should probably be compared with the Konqueror
1028 # (kcookiejar.cpp) and Mozilla implementations, but it's a
1029 # losing battle.
1030 i = domain.rfind(".")
1031 j = domain.rfind(".", 0, i)
1032 if j == 0: # domain like .foo.bar
1033 tld = domain[i+1:]
1034 sld = domain[j+1:i]
1035 if sld.lower() in ("co", "ac", "com", "edu", "org", "net",
1036 "gov", "mil", "int", "aero", "biz", "cat", "coop",
1037 "info", "jobs", "mobi", "museum", "name", "pro",
1038 "travel", "eu") and len(tld) == 2:
1039 # domain like .co.uk
1040 _debug(" country-code second level domain %s", domain)
1041 return False
1042 if domain.startswith("."):
1043 undotted_domain = domain[1:]
1044 else:
1045 undotted_domain = domain
1046 embedded_dots = (undotted_domain.find(".") >= 0)
1047 if not embedded_dots and not erhn.endswith(".local"):
1048 _debug(" non-local domain %s contains no embedded dot",
1049 domain)
1050 return False
1051 if cookie.version == 0:
1052 if (not (erhn.endswith(domain) or
1053 erhn.endswith(f"{undotted_domain}.local")) and
1054 (not erhn.startswith(".") and
1055 not ("."+erhn).endswith(domain))):
1056 _debug(" effective request-host %s (even with added "
1057 "initial dot) does not end with %s",
1058 erhn, domain)
1059 return False
1060 if (cookie.version > 0 or
1061 (self.strict_ns_domain & self.DomainRFC2965Match)):
1062 if not domain_match(erhn, domain):
1063 _debug(" effective request-host %s does not domain-match "
1064 "%s", erhn, domain)
1065 return False
1066 if (cookie.version > 0 or
1067 (self.strict_ns_domain & self.DomainStrictNoDots)):
1068 host_prefix = req_host[:-len(domain)]
1069 if (host_prefix.find(".") >= 0 and
1070 not IPV4_RE.search(req_host)):
1071 _debug(" host prefix %s for domain %s contains a dot",
1072 host_prefix, domain)
1073 return False
1074 return True
1075
1076 def set_ok_port(self, cookie, request):
1077 if cookie.port_specified:
1078 req_port = request_port(request)
1079 if req_port is None:
1080 req_port = "80"
1081 else:
1082 req_port = str(req_port)
1083 for p in cookie.port.split(","):
1084 try:
1085 int(p)
1086 except ValueError:
1087 _debug(" bad port %s (not numeric)", p)
1088 return False
1089 if p == req_port:
1090 break
1091 else:
1092 _debug(" request port (%s) not found in %s",
1093 req_port, cookie.port)
1094 return False
1095 return True
1096
1097 def return_ok(self, cookie, request):
1098 """
1099 If you override .return_ok(), be sure to call this method. If it
1100 returns false, so should your subclass (assuming your subclass wants to
1101 be more strict about which cookies to return).
1102
1103 """
1104 # Path has already been checked by .path_return_ok(), and domain
1105 # blocking done by .domain_return_ok().
1106 _debug(" - checking cookie %s=%s", cookie.name, cookie.value)
1107
1108 for n in "version", "verifiability", "secure", "expires", "port", "domain":
1109 fn_name = "return_ok_"+n
1110 fn = getattr(self, fn_name)
1111 if not fn(cookie, request):
1112 return False
1113 return True
1114
1115 def return_ok_version(self, cookie, request):
1116 if cookie.version > 0 and not self.rfc2965:
1117 _debug(" RFC 2965 cookies are switched off")
1118 return False
1119 elif cookie.version == 0 and not self.netscape:
1120 _debug(" Netscape cookies are switched off")
1121 return False
1122 return True
1123
1124 def return_ok_verifiability(self, cookie, request):
1125 if request.unverifiable and is_third_party(request):
1126 if cookie.version > 0 and self.strict_rfc2965_unverifiable:
1127 _debug(" third-party RFC 2965 cookie during unverifiable "
1128 "transaction")
1129 return False
1130 elif cookie.version == 0 and self.strict_ns_unverifiable:
1131 _debug(" third-party Netscape cookie during unverifiable "
1132 "transaction")
1133 return False
1134 return True
1135
1136 def return_ok_secure(self, cookie, request):
1137 if cookie.secure and request.type not in self.secure_protocols:
1138 _debug(" secure cookie with non-secure request")
1139 return False
1140 return True
1141
1142 def return_ok_expires(self, cookie, request):
1143 if cookie.is_expired(self._now):
1144 _debug(" cookie expired")
1145 return False
1146 return True
1147
1148 def return_ok_port(self, cookie, request):
1149 if cookie.port:
1150 req_port = request_port(request)
1151 if req_port is None:
1152 req_port = "80"
1153 for p in cookie.port.split(","):
1154 if p == req_port:
1155 break
1156 else:
1157 _debug(" request port %s does not match cookie port %s",
1158 req_port, cookie.port)
1159 return False
1160 return True
1161
1162 def return_ok_domain(self, cookie, request):
1163 req_host, erhn = eff_request_host(request)
1164 domain = cookie.domain
1165
1166 if domain and not domain.startswith("."):
1167 dotdomain = "." + domain
1168 else:
1169 dotdomain = domain
1170
1171 # strict check of non-domain cookies: Mozilla does this, MSIE5 doesn't
1172 if (cookie.version == 0 and
1173 (self.strict_ns_domain & self.DomainStrictNonDomain) and
1174 not cookie.domain_specified and domain != erhn):
1175 _debug(" cookie with unspecified domain does not string-compare "
1176 "equal to request domain")
1177 return False
1178
1179 if cookie.version > 0 and not domain_match(erhn, domain):
1180 _debug(" effective request-host name %s does not domain-match "
1181 "RFC 2965 cookie domain %s", erhn, domain)
1182 return False
1183 if cookie.version == 0 and not ("."+erhn).endswith(dotdomain):
1184 _debug(" request-host %s does not match Netscape cookie domain "
1185 "%s", req_host, domain)
1186 return False
1187 return True
1188
1189 def domain_return_ok(self, domain, request):
1190 # Liberal check of. This is here as an optimization to avoid
1191 # having to load lots of MSIE cookie files unless necessary.
1192 req_host, erhn = eff_request_host(request)
1193 if not req_host.startswith("."):
1194 req_host = "."+req_host
1195 if not erhn.startswith("."):
1196 erhn = "."+erhn
1197 if domain and not domain.startswith("."):
1198 dotdomain = "." + domain
1199 else:
1200 dotdomain = domain
1201 if not (req_host.endswith(dotdomain) or erhn.endswith(dotdomain)):
1202 #_debug(" request domain %s does not match cookie domain %s",
1203 # req_host, domain)
1204 return False
1205
1206 if self.is_blocked(domain):
1207 _debug(" domain %s is in user block-list", domain)
1208 return False
1209 if self.is_not_allowed(domain):
1210 _debug(" domain %s is not in user allow-list", domain)
1211 return False
1212
1213 return True
1214
1215 def path_return_ok(self, path, request):
1216 _debug("- checking cookie path=%s", path)
1217 req_path = request_path(request)
1218 pathlen = len(path)
1219 if req_path == path:
1220 return True
1221 elif (req_path.startswith(path) and
1222 (path.endswith("/") or req_path[pathlen:pathlen+1] == "/")):
1223 return True
1224
1225 _debug(" %s does not path-match %s", req_path, path)
1226 return False
1227
1228 def deepvalues(mapping):
1229 """Iterates over nested mapping, depth-first"""
1230 for obj in list(mapping.values()):
1231 mapping = False
1232 try:
1233 obj.items
1234 except AttributeError:
1235 pass
1236 else:
1237 mapping = True
1238 yield from deepvalues(obj)
1239 if not mapping:
1240 yield obj
1241
1242
1243 # Used as second parameter to dict.get() method, to distinguish absent
1244 # dict key from one with a None value.
1245 class Absent: pass
1246
1247 class CookieJar:
1248 """Collection of HTTP cookies.
1249
1250 You may not need to know about this class: try
1251 urllib.request.build_opener(HTTPCookieProcessor).open(url).
1252 """
1253
1254 non_word_re = re.compile(r"\W")
1255 quote_re = re.compile(r"([\"\\])")
1256 strict_domain_re = re.compile(r"\.?[^.]*")
1257 domain_re = re.compile(r"[^.]*")
1258 dots_re = re.compile(r"^\.+")
1259
1260 magic_re = re.compile(r"^\#LWP-Cookies-(\d+\.\d+)", re.ASCII)
1261
1262 def __init__(self, policy=None):
1263 if policy is None:
1264 policy = DefaultCookiePolicy()
1265 self._policy = policy
1266
1267 self._cookies_lock = _threading.RLock()
1268 self._cookies = {}
1269
1270 def set_policy(self, policy):
1271 self._policy = policy
1272
1273 def _cookies_for_domain(self, domain, request):
1274 cookies = []
1275 if not self._policy.domain_return_ok(domain, request):
1276 return []
1277 _debug("Checking %s for cookies to return", domain)
1278 cookies_by_path = self._cookies[domain]
1279 for path in cookies_by_path.keys():
1280 if not self._policy.path_return_ok(path, request):
1281 continue
1282 cookies_by_name = cookies_by_path[path]
1283 for cookie in cookies_by_name.values():
1284 if not self._policy.return_ok(cookie, request):
1285 _debug(" not returning cookie")
1286 continue
1287 _debug(" it's a match")
1288 cookies.append(cookie)
1289 return cookies
1290
1291 def _cookies_for_request(self, request):
1292 """Return a list of cookies to be returned to server."""
1293 cookies = []
1294 for domain in self._cookies.keys():
1295 cookies.extend(self._cookies_for_domain(domain, request))
1296 return cookies
1297
1298 def _cookie_attrs(self, cookies):
1299 """Return a list of cookie-attributes to be returned to server.
1300
1301 like ['foo="bar"; $Path="/"', ...]
1302
1303 The $Version attribute is also added when appropriate (currently only
1304 once per request).
1305
1306 """
1307 # add cookies in order of most specific (ie. longest) path first
1308 cookies.sort(key=lambda a: len(a.path), reverse=True)
1309
1310 version_set = False
1311
1312 attrs = []
1313 for cookie in cookies:
1314 # set version of Cookie header
1315 # XXX
1316 # What should it be if multiple matching Set-Cookie headers have
1317 # different versions themselves?
1318 # Answer: there is no answer; was supposed to be settled by
1319 # RFC 2965 errata, but that may never appear...
1320 version = cookie.version
1321 if not version_set:
1322 version_set = True
1323 if version > 0:
1324 attrs.append("$Version=%s" % version)
1325
1326 # quote cookie value if necessary
1327 # (not for Netscape protocol, which already has any quotes
1328 # intact, due to the poorly-specified Netscape Cookie: syntax)
1329 if ((cookie.value is not None) and
1330 self.non_word_re.search(cookie.value) and version > 0):
1331 value = self.quote_re.sub(r"\\\1", cookie.value)
1332 else:
1333 value = cookie.value
1334
1335 # add cookie-attributes to be returned in Cookie header
1336 if cookie.value is None:
1337 attrs.append(cookie.name)
1338 else:
1339 attrs.append("%s=%s" % (cookie.name, value))
1340 if version > 0:
1341 if cookie.path_specified:
1342 attrs.append('$Path="%s"' % cookie.path)
1343 if cookie.domain.startswith("."):
1344 domain = cookie.domain
1345 if (not cookie.domain_initial_dot and
1346 domain.startswith(".")):
1347 domain = domain[1:]
1348 attrs.append('$Domain="%s"' % domain)
1349 if cookie.port is not None:
1350 p = "$Port"
1351 if cookie.port_specified:
1352 p = p + ('="%s"' % cookie.port)
1353 attrs.append(p)
1354
1355 return attrs
1356
1357 def add_cookie_header(self, request):
1358 """Add correct Cookie: header to request (urllib.request.Request object).
1359
1360 The Cookie2 header is also added unless policy.hide_cookie2 is true.
1361
1362 """
1363 _debug("add_cookie_header")
1364 self._cookies_lock.acquire()
1365 try:
1366
1367 self._policy._now = self._now = int(time.time())
1368
1369 cookies = self._cookies_for_request(request)
1370
1371 attrs = self._cookie_attrs(cookies)
1372 if attrs:
1373 if not request.has_header("Cookie"):
1374 request.add_unredirected_header(
1375 "Cookie", "; ".join(attrs))
1376
1377 # if necessary, advertise that we know RFC 2965
1378 if (self._policy.rfc2965 and not self._policy.hide_cookie2 and
1379 not request.has_header("Cookie2")):
1380 for cookie in cookies:
1381 if cookie.version != 1:
1382 request.add_unredirected_header("Cookie2", '$Version="1"')
1383 break
1384
1385 finally:
1386 self._cookies_lock.release()
1387
1388 self.clear_expired_cookies()
1389
1390 def _normalized_cookie_tuples(self, attrs_set):
1391 """Return list of tuples containing normalised cookie information.
1392
1393 attrs_set is the list of lists of key,value pairs extracted from
1394 the Set-Cookie or Set-Cookie2 headers.
1395
1396 Tuples are name, value, standard, rest, where name and value are the
1397 cookie name and value, standard is a dictionary containing the standard
1398 cookie-attributes (discard, secure, version, expires or max-age,
1399 domain, path and port) and rest is a dictionary containing the rest of
1400 the cookie-attributes.
1401
1402 """
1403 cookie_tuples = []
1404
1405 boolean_attrs = "discard", "secure"
1406 value_attrs = ("version",
1407 "expires", "max-age",
1408 "domain", "path", "port",
1409 "comment", "commenturl")
1410
1411 for cookie_attrs in attrs_set:
1412 name, value = cookie_attrs[0]
1413
1414 # Build dictionary of standard cookie-attributes (standard) and
1415 # dictionary of other cookie-attributes (rest).
1416
1417 # Note: expiry time is normalised to seconds since epoch. V0
1418 # cookies should have the Expires cookie-attribute, and V1 cookies
1419 # should have Max-Age, but since V1 includes RFC 2109 cookies (and
1420 # since V0 cookies may be a mish-mash of Netscape and RFC 2109), we
1421 # accept either (but prefer Max-Age).
1422 max_age_set = False
1423
1424 bad_cookie = False
1425
1426 standard = {}
1427 rest = {}
1428 for k, v in cookie_attrs[1:]:
1429 lc = k.lower()
1430 # don't lose case distinction for unknown fields
1431 if lc in value_attrs or lc in boolean_attrs:
1432 k = lc
1433 if k in boolean_attrs and v is None:
1434 # boolean cookie-attribute is present, but has no value
1435 # (like "discard", rather than "port=80")
1436 v = True
1437 if k in standard:
1438 # only first value is significant
1439 continue
1440 if k == "domain":
1441 if v is None:
1442 _debug(" missing value for domain attribute")
1443 bad_cookie = True
1444 break
1445 # RFC 2965 section 3.3.3
1446 v = v.lower()
1447 if k == "expires":
1448 if max_age_set:
1449 # Prefer max-age to expires (like Mozilla)
1450 continue
1451 if v is None:
1452 _debug(" missing or invalid value for expires "
1453 "attribute: treating as session cookie")
1454 continue
1455 if k == "max-age":
1456 max_age_set = True
1457 try:
1458 v = int(v)
1459 except ValueError:
1460 _debug(" missing or invalid (non-numeric) value for "
1461 "max-age attribute")
1462 bad_cookie = True
1463 break
1464 # convert RFC 2965 Max-Age to seconds since epoch
1465 # XXX Strictly you're supposed to follow RFC 2616
1466 # age-calculation rules. Remember that zero Max-Age
1467 # is a request to discard (old and new) cookie, though.
1468 k = "expires"
1469 v = self._now + v
1470 if (k in value_attrs) or (k in boolean_attrs):
1471 if (v is None and
1472 k not in ("port", "comment", "commenturl")):
1473 _debug(" missing value for %s attribute" % k)
1474 bad_cookie = True
1475 break
1476 standard[k] = v
1477 else:
1478 rest[k] = v
1479
1480 if bad_cookie:
1481 continue
1482
1483 cookie_tuples.append((name, value, standard, rest))
1484
1485 return cookie_tuples
1486
1487 def _cookie_from_cookie_tuple(self, tup, request):
1488 # standard is dict of standard cookie-attributes, rest is dict of the
1489 # rest of them
1490 name, value, standard, rest = tup
1491
1492 domain = standard.get("domain", Absent)
1493 path = standard.get("path", Absent)
1494 port = standard.get("port", Absent)
1495 expires = standard.get("expires", Absent)
1496
1497 # set the easy defaults
1498 version = standard.get("version", None)
1499 if version is not None:
1500 try:
1501 version = int(version)
1502 except ValueError:
1503 return None # invalid version, ignore cookie
1504 secure = standard.get("secure", False)
1505 # (discard is also set if expires is Absent)
1506 discard = standard.get("discard", False)
1507 comment = standard.get("comment", None)
1508 comment_url = standard.get("commenturl", None)
1509
1510 # set default path
1511 if path is not Absent and path != "":
1512 path_specified = True
1513 path = escape_path(path)
1514 else:
1515 path_specified = False
1516 path = request_path(request)
1517 i = path.rfind("/")
1518 if i != -1:
1519 if version == 0:
1520 # Netscape spec parts company from reality here
1521 path = path[:i]
1522 else:
1523 path = path[:i+1]
1524 if len(path) == 0: path = "/"
1525
1526 # set default domain
1527 domain_specified = domain is not Absent
1528 # but first we have to remember whether it starts with a dot
1529 domain_initial_dot = False
1530 if domain_specified:
1531 domain_initial_dot = bool(domain.startswith("."))
1532 if domain is Absent:
1533 req_host, erhn = eff_request_host(request)
1534 domain = erhn
1535 elif not domain.startswith("."):
1536 domain = "."+domain
1537
1538 # set default port
1539 port_specified = False
1540 if port is not Absent:
1541 if port is None:
1542 # Port attr present, but has no value: default to request port.
1543 # Cookie should then only be sent back on that port.
1544 port = request_port(request)
1545 else:
1546 port_specified = True
1547 port = re.sub(r"\s+", "", port)
1548 else:
1549 # No port attr present. Cookie can be sent back on any port.
1550 port = None
1551
1552 # set default expires and discard
1553 if expires is Absent:
1554 expires = None
1555 discard = True
1556 elif expires <= self._now:
1557 # Expiry date in past is request to delete cookie. This can't be
1558 # in DefaultCookiePolicy, because can't delete cookies there.
1559 try:
1560 self.clear(domain, path, name)
1561 except KeyError:
1562 pass
1563 _debug("Expiring cookie, domain='%s', path='%s', name='%s'",
1564 domain, path, name)
1565 return None
1566
1567 return Cookie(version,
1568 name, value,
1569 port, port_specified,
1570 domain, domain_specified, domain_initial_dot,
1571 path, path_specified,
1572 secure,
1573 expires,
1574 discard,
1575 comment,
1576 comment_url,
1577 rest)
1578
1579 def _cookies_from_attrs_set(self, attrs_set, request):
1580 cookie_tuples = self._normalized_cookie_tuples(attrs_set)
1581
1582 cookies = []
1583 for tup in cookie_tuples:
1584 cookie = self._cookie_from_cookie_tuple(tup, request)
1585 if cookie: cookies.append(cookie)
1586 return cookies
1587
1588 def _process_rfc2109_cookies(self, cookies):
1589 rfc2109_as_ns = getattr(self._policy, 'rfc2109_as_netscape', None)
1590 if rfc2109_as_ns is None:
1591 rfc2109_as_ns = not self._policy.rfc2965
1592 for cookie in cookies:
1593 if cookie.version == 1:
1594 cookie.rfc2109 = True
1595 if rfc2109_as_ns:
1596 # treat 2109 cookies as Netscape cookies rather than
1597 # as RFC2965 cookies
1598 cookie.version = 0
1599
1600 def make_cookies(self, response, request):
1601 """Return sequence of Cookie objects extracted from response object."""
1602 # get cookie-attributes for RFC 2965 and Netscape protocols
1603 headers = response.info()
1604 rfc2965_hdrs = headers.get_all("Set-Cookie2", [])
1605 ns_hdrs = headers.get_all("Set-Cookie", [])
1606 self._policy._now = self._now = int(time.time())
1607
1608 rfc2965 = self._policy.rfc2965
1609 netscape = self._policy.netscape
1610
1611 if ((not rfc2965_hdrs and not ns_hdrs) or
1612 (not ns_hdrs and not rfc2965) or
1613 (not rfc2965_hdrs and not netscape) or
1614 (not netscape and not rfc2965)):
1615 return [] # no relevant cookie headers: quick exit
1616
1617 try:
1618 cookies = self._cookies_from_attrs_set(
1619 split_header_words(rfc2965_hdrs), request)
1620 except Exception:
1621 _warn_unhandled_exception()
1622 cookies = []
1623
1624 if ns_hdrs and netscape:
1625 try:
1626 # RFC 2109 and Netscape cookies
1627 ns_cookies = self._cookies_from_attrs_set(
1628 parse_ns_headers(ns_hdrs), request)
1629 except Exception:
1630 _warn_unhandled_exception()
1631 ns_cookies = []
1632 self._process_rfc2109_cookies(ns_cookies)
1633
1634 # Look for Netscape cookies (from Set-Cookie headers) that match
1635 # corresponding RFC 2965 cookies (from Set-Cookie2 headers).
1636 # For each match, keep the RFC 2965 cookie and ignore the Netscape
1637 # cookie (RFC 2965 section 9.1). Actually, RFC 2109 cookies are
1638 # bundled in with the Netscape cookies for this purpose, which is
1639 # reasonable behaviour.
1640 if rfc2965:
1641 lookup = {}
1642 for cookie in cookies:
1643 lookup[(cookie.domain, cookie.path, cookie.name)] = None
1644
1645 def no_matching_rfc2965(ns_cookie, lookup=lookup):
1646 key = ns_cookie.domain, ns_cookie.path, ns_cookie.name
1647 return key not in lookup
1648 ns_cookies = filter(no_matching_rfc2965, ns_cookies)
1649
1650 if ns_cookies:
1651 cookies.extend(ns_cookies)
1652
1653 return cookies
1654
1655 def set_cookie_if_ok(self, cookie, request):
1656 """Set a cookie if policy says it's OK to do so."""
1657 self._cookies_lock.acquire()
1658 try:
1659 self._policy._now = self._now = int(time.time())
1660
1661 if self._policy.set_ok(cookie, request):
1662 self.set_cookie(cookie)
1663
1664
1665 finally:
1666 self._cookies_lock.release()
1667
1668 def set_cookie(self, cookie):
1669 """Set a cookie, without checking whether or not it should be set."""
1670 c = self._cookies
1671 self._cookies_lock.acquire()
1672 try:
1673 if cookie.domain not in c: c[cookie.domain] = {}
1674 c2 = c[cookie.domain]
1675 if cookie.path not in c2: c2[cookie.path] = {}
1676 c3 = c2[cookie.path]
1677 c3[cookie.name] = cookie
1678 finally:
1679 self._cookies_lock.release()
1680
1681 def extract_cookies(self, response, request):
1682 """Extract cookies from response, where allowable given the request."""
1683 _debug("extract_cookies: %s", response.info())
1684 self._cookies_lock.acquire()
1685 try:
1686 for cookie in self.make_cookies(response, request):
1687 if self._policy.set_ok(cookie, request):
1688 _debug(" setting cookie: %s", cookie)
1689 self.set_cookie(cookie)
1690 finally:
1691 self._cookies_lock.release()
1692
1693 def clear(self, domain=None, path=None, name=None):
1694 """Clear some cookies.
1695
1696 Invoking this method without arguments will clear all cookies. If
1697 given a single argument, only cookies belonging to that domain will be
1698 removed. If given two arguments, cookies belonging to the specified
1699 path within that domain are removed. If given three arguments, then
1700 the cookie with the specified name, path and domain is removed.
1701
1702 Raises KeyError if no matching cookie exists.
1703
1704 """
1705 if name is not None:
1706 if (domain is None) or (path is None):
1707 raise ValueError(
1708 "domain and path must be given to remove a cookie by name")
1709 del self._cookies[domain][path][name]
1710 elif path is not None:
1711 if domain is None:
1712 raise ValueError(
1713 "domain must be given to remove cookies by path")
1714 del self._cookies[domain][path]
1715 elif domain is not None:
1716 del self._cookies[domain]
1717 else:
1718 self._cookies = {}
1719
1720 def clear_session_cookies(self):
1721 """Discard all session cookies.
1722
1723 Note that the .save() method won't save session cookies anyway, unless
1724 you ask otherwise by passing a true ignore_discard argument.
1725
1726 """
1727 self._cookies_lock.acquire()
1728 try:
1729 for cookie in self:
1730 if cookie.discard:
1731 self.clear(cookie.domain, cookie.path, cookie.name)
1732 finally:
1733 self._cookies_lock.release()
1734
1735 def clear_expired_cookies(self):
1736 """Discard all expired cookies.
1737
1738 You probably don't need to call this method: expired cookies are never
1739 sent back to the server (provided you're using DefaultCookiePolicy),
1740 this method is called by CookieJar itself every so often, and the
1741 .save() method won't save expired cookies anyway (unless you ask
1742 otherwise by passing a true ignore_expires argument).
1743
1744 """
1745 self._cookies_lock.acquire()
1746 try:
1747 now = time.time()
1748 for cookie in self:
1749 if cookie.is_expired(now):
1750 self.clear(cookie.domain, cookie.path, cookie.name)
1751 finally:
1752 self._cookies_lock.release()
1753
1754 def __iter__(self):
1755 return deepvalues(self._cookies)
1756
1757 def __len__(self):
1758 """Return number of contained cookies."""
1759 i = 0
1760 for cookie in self: i = i + 1
1761 return i
1762
1763 def __repr__(self):
1764 r = []
1765 for cookie in self: r.append(repr(cookie))
1766 return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
1767
1768 def __str__(self):
1769 r = []
1770 for cookie in self: r.append(str(cookie))
1771 return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
1772
1773
1774 # derives from OSError for backwards-compatibility with Python 2.4.0
1775 class LoadError(OSError): pass
1776
1777 class FileCookieJar(CookieJar):
1778 """CookieJar that can be loaded from and saved to a file."""
1779
1780 def __init__(self, filename=None, delayload=False, policy=None):
1781 """
1782 Cookies are NOT loaded from the named file until either the .load() or
1783 .revert() method is called.
1784
1785 """
1786 CookieJar.__init__(self, policy)
1787 if filename is not None:
1788 filename = os.fspath(filename)
1789 self.filename = filename
1790 self.delayload = bool(delayload)
1791
1792 def save(self, filename=None, ignore_discard=False, ignore_expires=False):
1793 """Save cookies to a file."""
1794 raise NotImplementedError()
1795
1796 def load(self, filename=None, ignore_discard=False, ignore_expires=False):
1797 """Load cookies from a file."""
1798 if filename is None:
1799 if self.filename is not None: filename = self.filename
1800 else: raise ValueError(MISSING_FILENAME_TEXT)
1801
1802 with open(filename) as f:
1803 self._really_load(f, filename, ignore_discard, ignore_expires)
1804
1805 def revert(self, filename=None,
1806 ignore_discard=False, ignore_expires=False):
1807 """Clear all cookies and reload cookies from a saved file.
1808
1809 Raises LoadError (or OSError) if reversion is not successful; the
1810 object's state will not be altered if this happens.
1811
1812 """
1813 if filename is None:
1814 if self.filename is not None: filename = self.filename
1815 else: raise ValueError(MISSING_FILENAME_TEXT)
1816
1817 self._cookies_lock.acquire()
1818 try:
1819
1820 old_state = copy.deepcopy(self._cookies)
1821 self._cookies = {}
1822 try:
1823 self.load(filename, ignore_discard, ignore_expires)
1824 except OSError:
1825 self._cookies = old_state
1826 raise
1827
1828 finally:
1829 self._cookies_lock.release()
1830
1831
1832 def lwp_cookie_str(cookie):
1833 """Return string representation of Cookie in the LWP cookie file format.
1834
1835 Actually, the format is extended a bit -- see module docstring.
1836
1837 """
1838 h = [(cookie.name, cookie.value),
1839 ("path", cookie.path),
1840 ("domain", cookie.domain)]
1841 if cookie.port is not None: h.append(("port", cookie.port))
1842 if cookie.path_specified: h.append(("path_spec", None))
1843 if cookie.port_specified: h.append(("port_spec", None))
1844 if cookie.domain_initial_dot: h.append(("domain_dot", None))
1845 if cookie.secure: h.append(("secure", None))
1846 if cookie.expires: h.append(("expires",
1847 time2isoz(float(cookie.expires))))
1848 if cookie.discard: h.append(("discard", None))
1849 if cookie.comment: h.append(("comment", cookie.comment))
1850 if cookie.comment_url: h.append(("commenturl", cookie.comment_url))
1851
1852 keys = sorted(cookie._rest.keys())
1853 for k in keys:
1854 h.append((k, str(cookie._rest[k])))
1855
1856 h.append(("version", str(cookie.version)))
1857
1858 return join_header_words([h])
1859
1860 class LWPCookieJar(FileCookieJar):
1861 """
1862 The LWPCookieJar saves a sequence of "Set-Cookie3" lines.
1863 "Set-Cookie3" is the format used by the libwww-perl library, not known
1864 to be compatible with any browser, but which is easy to read and
1865 doesn't lose information about RFC 2965 cookies.
1866
1867 Additional methods
1868
1869 as_lwp_str(ignore_discard=True, ignore_expired=True)
1870
1871 """
1872
1873 def as_lwp_str(self, ignore_discard=True, ignore_expires=True):
1874 """Return cookies as a string of "\\n"-separated "Set-Cookie3" headers.
1875
1876 ignore_discard and ignore_expires: see docstring for FileCookieJar.save
1877
1878 """
1879 now = time.time()
1880 r = []
1881 for cookie in self:
1882 if not ignore_discard and cookie.discard:
1883 continue
1884 if not ignore_expires and cookie.is_expired(now):
1885 continue
1886 r.append("Set-Cookie3: %s" % lwp_cookie_str(cookie))
1887 return "\n".join(r+[""])
1888
1889 def save(self, filename=None, ignore_discard=False, ignore_expires=False):
1890 if filename is None:
1891 if self.filename is not None: filename = self.filename
1892 else: raise ValueError(MISSING_FILENAME_TEXT)
1893
1894 with os.fdopen(
1895 os.open(filename, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600),
1896 'w',
1897 ) as f:
1898 # There really isn't an LWP Cookies 2.0 format, but this indicates
1899 # that there is extra information in here (domain_dot and
1900 # port_spec) while still being compatible with libwww-perl, I hope.
1901 f.write("#LWP-Cookies-2.0\n")
1902 f.write(self.as_lwp_str(ignore_discard, ignore_expires))
1903
1904 def _really_load(self, f, filename, ignore_discard, ignore_expires):
1905 magic = f.readline()
1906 if not self.magic_re.search(magic):
1907 msg = ("%r does not look like a Set-Cookie3 (LWP) format "
1908 "file" % filename)
1909 raise LoadError(msg)
1910
1911 now = time.time()
1912
1913 header = "Set-Cookie3:"
1914 boolean_attrs = ("port_spec", "path_spec", "domain_dot",
1915 "secure", "discard")
1916 value_attrs = ("version",
1917 "port", "path", "domain",
1918 "expires",
1919 "comment", "commenturl")
1920
1921 try:
1922 while (line := f.readline()) != "":
1923 if not line.startswith(header):
1924 continue
1925 line = line[len(header):].strip()
1926
1927 for data in split_header_words([line]):
1928 name, value = data[0]
1929 standard = {}
1930 rest = {}
1931 for k in boolean_attrs:
1932 standard[k] = False
1933 for k, v in data[1:]:
1934 if k is not None:
1935 lc = k.lower()
1936 else:
1937 lc = None
1938 # don't lose case distinction for unknown fields
1939 if (lc in value_attrs) or (lc in boolean_attrs):
1940 k = lc
1941 if k in boolean_attrs:
1942 if v is None: v = True
1943 standard[k] = v
1944 elif k in value_attrs:
1945 standard[k] = v
1946 else:
1947 rest[k] = v
1948
1949 h = standard.get
1950 expires = h("expires")
1951 discard = h("discard")
1952 if expires is not None:
1953 expires = iso2time(expires)
1954 if expires is None:
1955 discard = True
1956 domain = h("domain")
1957 domain_specified = domain.startswith(".")
1958 c = Cookie(h("version"), name, value,
1959 h("port"), h("port_spec"),
1960 domain, domain_specified, h("domain_dot"),
1961 h("path"), h("path_spec"),
1962 h("secure"),
1963 expires,
1964 discard,
1965 h("comment"),
1966 h("commenturl"),
1967 rest)
1968 if not ignore_discard and c.discard:
1969 continue
1970 if not ignore_expires and c.is_expired(now):
1971 continue
1972 self.set_cookie(c)
1973 except OSError:
1974 raise
1975 except Exception:
1976 _warn_unhandled_exception()
1977 raise LoadError("invalid Set-Cookie3 format file %r: %r" %
1978 (filename, line))
1979
1980
1981 class MozillaCookieJar(FileCookieJar):
1982 """
1983
1984 WARNING: you may want to backup your browser's cookies file if you use
1985 this class to save cookies. I *think* it works, but there have been
1986 bugs in the past!
1987
1988 This class differs from CookieJar only in the format it uses to save and
1989 load cookies to and from a file. This class uses the Mozilla/Netscape
1990 'cookies.txt' format. curl and lynx use this file format, too.
1991
1992 Don't expect cookies saved while the browser is running to be noticed by
1993 the browser (in fact, Mozilla on unix will overwrite your saved cookies if
1994 you change them on disk while it's running; on Windows, you probably can't
1995 save at all while the browser is running).
1996
1997 Note that the Mozilla/Netscape format will downgrade RFC2965 cookies to
1998 Netscape cookies on saving.
1999
2000 In particular, the cookie version and port number information is lost,
2001 together with information about whether or not Path, Port and Discard were
2002 specified by the Set-Cookie2 (or Set-Cookie) header, and whether or not the
2003 domain as set in the HTTP header started with a dot (yes, I'm aware some
2004 domains in Netscape files start with a dot and some don't -- trust me, you
2005 really don't want to know any more about this).
2006
2007 Note that though Mozilla and Netscape use the same format, they use
2008 slightly different headers. The class saves cookies using the Netscape
2009 header by default (Mozilla can cope with that).
2010
2011 """
2012
2013 def _really_load(self, f, filename, ignore_discard, ignore_expires):
2014 now = time.time()
2015
2016 if not NETSCAPE_MAGIC_RGX.match(f.readline()):
2017 raise LoadError(
2018 "%r does not look like a Netscape format cookies file" %
2019 filename)
2020
2021 try:
2022 while (line := f.readline()) != "":
2023 rest = {}
2024
2025 # httponly is a cookie flag as defined in rfc6265
2026 # when encoded in a netscape cookie file,
2027 # the line is prepended with "#HttpOnly_"
2028 if line.startswith(HTTPONLY_PREFIX):
2029 rest[HTTPONLY_ATTR] = ""
2030 line = line[len(HTTPONLY_PREFIX):]
2031
2032 # last field may be absent, so keep any trailing tab
2033 if line.endswith("\n"): line = line[:-1]
2034
2035 # skip comments and blank lines XXX what is $ for?
2036 if (line.strip().startswith(("#", "$")) or
2037 line.strip() == ""):
2038 continue
2039
2040 domain, domain_specified, path, secure, expires, name, value = \
2041 line.split("\t")
2042 secure = (secure == "TRUE")
2043 domain_specified = (domain_specified == "TRUE")
2044 if name == "":
2045 # cookies.txt regards 'Set-Cookie: foo' as a cookie
2046 # with no name, whereas http.cookiejar regards it as a
2047 # cookie with no value.
2048 name = value
2049 value = None
2050
2051 initial_dot = domain.startswith(".")
2052 assert domain_specified == initial_dot
2053
2054 discard = False
2055 if expires == "":
2056 expires = None
2057 discard = True
2058
2059 # assume path_specified is false
2060 c = Cookie(0, name, value,
2061 None, False,
2062 domain, domain_specified, initial_dot,
2063 path, False,
2064 secure,
2065 expires,
2066 discard,
2067 None,
2068 None,
2069 rest)
2070 if not ignore_discard and c.discard:
2071 continue
2072 if not ignore_expires and c.is_expired(now):
2073 continue
2074 self.set_cookie(c)
2075
2076 except OSError:
2077 raise
2078 except Exception:
2079 _warn_unhandled_exception()
2080 raise LoadError("invalid Netscape format cookies file %r: %r" %
2081 (filename, line))
2082
2083 def save(self, filename=None, ignore_discard=False, ignore_expires=False):
2084 if filename is None:
2085 if self.filename is not None: filename = self.filename
2086 else: raise ValueError(MISSING_FILENAME_TEXT)
2087
2088 with os.fdopen(
2089 os.open(filename, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600),
2090 'w',
2091 ) as f:
2092 f.write(NETSCAPE_HEADER_TEXT)
2093 now = time.time()
2094 for cookie in self:
2095 domain = cookie.domain
2096 if not ignore_discard and cookie.discard:
2097 continue
2098 if not ignore_expires and cookie.is_expired(now):
2099 continue
2100 if cookie.secure: secure = "TRUE"
2101 else: secure = "FALSE"
2102 if domain.startswith("."): initial_dot = "TRUE"
2103 else: initial_dot = "FALSE"
2104 if cookie.expires is not None:
2105 expires = str(cookie.expires)
2106 else:
2107 expires = ""
2108 if cookie.value is None:
2109 # cookies.txt regards 'Set-Cookie: foo' as a cookie
2110 # with no name, whereas http.cookiejar regards it as a
2111 # cookie with no value.
2112 name = ""
2113 value = cookie.name
2114 else:
2115 name = cookie.name
2116 value = cookie.value
2117 if cookie.has_nonstandard_attr(HTTPONLY_ATTR):
2118 domain = HTTPONLY_PREFIX + domain
2119 f.write(
2120 "\t".join([domain, initial_dot, cookie.path,
2121 secure, expires, name, value])+
2122 "\n")