| 1 |
""" |
| 2 |
Utilities to parse patterns and match URLs using them. |
| 3 |
""" |
| 4 |
|
| 5 |
from __future__ import annotations |
| 6 |
|
| 7 |
import ipaddress |
| 8 |
import re |
| 9 |
import warnings |
| 10 |
from functools import lru_cache |
| 11 |
from re import Pattern |
| 12 |
from typing import NamedTuple |
| 13 |
from urllib.parse import parse_qs, urlparse |
| 14 |
|
| 15 |
from url_matcher.util import get_domain |
| 16 |
|
| 17 |
|
| 18 |
def get_pattern_domain(pattern: str) -> str | None: |
| 19 |
""" |
| 20 |
Returns the domain of the pattern if any. |
| 21 |
|
| 22 |
>>> get_pattern_domain("") |
| 23 |
|
| 24 |
>>> get_pattern_domain("/") |
| 25 |
|
| 26 |
>>> get_pattern_domain("dom") |
| 27 |
'dom' |
| 28 |
>>> get_pattern_domain("DOM") |
| 29 |
'dom' |
| 30 |
>>> get_pattern_domain("dom:80") |
| 31 |
'dom' |
| 32 |
>>> get_pattern_domain("http://dom:80") |
| 33 |
'dom' |
| 34 |
>>> get_pattern_domain("http://dom/a_path") |
| 35 |
'dom' |
| 36 |
""" |
| 37 |
parsed = pattern_parse(pattern) |
| 38 |
if parsed.netloc: |
| 39 |
return get_domain(f"//{parsed.netloc}").lower() |
| 40 |
return None |
| 41 |
|
| 42 |
|
| 43 |
def pattern_to_url(pattern: str) -> str: |
| 44 |
""" |
| 45 |
Required for urlparse to recognize the domain in patterns |
| 46 |
like example.com/path |
| 47 |
|
| 48 |
>>> pattern_to_url("example.com/") |
| 49 |
'//example.com/' |
| 50 |
>>> pattern_to_url("example.com") |
| 51 |
'//example.com' |
| 52 |
>>> pattern_to_url("https://example.com") |
| 53 |
'https://example.com' |
| 54 |
>>> pattern_to_url("MySchema4+.-://example.com") |
| 55 |
'MySchema4+.-://example.com' |
| 56 |
>>> pattern_to_url("//example.com") |
| 57 |
'////example.com' |
| 58 |
""" |
| 59 |
# As defined in https://datatracker.ietf.org/doc/html/rfc3986#section-3.1 |
| 60 |
has_scheme = re.search(r"^([a-zA-Z][a-zA-Z0-9.+-]*:)?//", pattern) |
| 61 |
if not has_scheme: |
| 62 |
pattern = f"//{pattern}" |
| 63 |
elif pattern.startswith("//"): |
| 64 |
# This is required because urlparse("//example.com").netloc == "//example.com" |
| 65 |
# but instead we want it to be parsed into the the path. We achieve it by appending |
| 66 |
# two more slashes |
| 67 |
pattern = f"//{pattern}" |
| 68 |
return pattern |
| 69 |
|
| 70 |
|
| 71 |
class ParseTuple(NamedTuple): |
| 72 |
scheme: str |
| 73 |
netloc: str |
| 74 |
path: str |
| 75 |
query: str |
| 76 |
fragment: str |
| 77 |
|
| 78 |
|
| 79 |
@lru_cache(30) |
| 80 |
def pattern_parse(pattern: str) -> ParseTuple: |
| 81 |
""" |
| 82 |
Parses the pattern to a named tuple (scheme, netloc, path, query, fragment) |
| 83 |
>>> pattern_parse("example.com") |
| 84 |
ParseTuple(scheme='', netloc='example.com', path='', query='', fragment='') |
| 85 |
>>> pattern_parse("//example.com/path;this_is_also_path") |
| 86 |
ParseTuple(scheme='', netloc='', path='//example.com/path;this_is_also_path', query='', fragment='') |
| 87 |
""" |
| 88 |
pattern = pattern_to_url(pattern) |
| 89 |
return _urlparse(pattern) |
| 90 |
|
| 91 |
|
| 92 |
def _urlparse(url: str) -> ParseTuple: |
| 93 |
""" |
| 94 |
Returns a named tuple (scheme, netloc, path, query, fragment) |
| 95 |
where path and params are joined together into path and |
| 96 |
some other elements are normalized. |
| 97 |
|
| 98 |
>>> _urlparse("scheme://example.com/path;params?query=23#fragment") |
| 99 |
ParseTuple(scheme='scheme', netloc='example.com', path='/path;params', query='query=23', fragment='fragment') |
| 100 |
>>> _urlparse("http://example.com:80/path") |
| 101 |
ParseTuple(scheme='http', netloc='example.com', path='/path', query='', fragment='') |
| 102 |
""" |
| 103 |
scheme, netloc, path, params, query, fragment = urlparse(url) |
| 104 |
path = _join_path_and_params(path, params) |
| 105 |
scheme, netloc = normalize_netloc_and_schema(scheme, netloc) |
| 106 |
return ParseTuple(scheme, netloc, path, query, fragment) |
| 107 |
|
| 108 |
|
| 109 |
def _wildcard_re_escape(text: str) -> str: |
| 110 |
return re.escape(text).replace("\\*", ".*") |
| 111 |
|
| 112 |
|
| 113 |
def _join_path_and_params(path: str, params: str) -> str: |
| 114 |
if params: |
| 115 |
return f"{path};{params}" |
| 116 |
return path |
| 117 |
|
| 118 |
|
| 119 |
def normalize_netloc_and_schema(schema: str, netloc: str) -> tuple[str, str]: |
| 120 |
""" |
| 121 |
Removes 80 or 443 port when obvious. Deduces http or https when the port is provided |
| 122 |
|
| 123 |
>>> normalize_netloc_and_schema("http", "example.com:80") |
| 124 |
('http', 'example.com') |
| 125 |
>>> normalize_netloc_and_schema("http", "example.com:80") |
| 126 |
('http', 'example.com') |
| 127 |
>>> normalize_netloc_and_schema("http", "example.com:443") |
| 128 |
('http', 'example.com:443') |
| 129 |
>>> normalize_netloc_and_schema("https", "example.com:443") |
| 130 |
('https', 'example.com') |
| 131 |
>>> normalize_netloc_and_schema("", "example.com:80") |
| 132 |
('http', 'example.com') |
| 133 |
>>> normalize_netloc_and_schema("", "example.com:443") |
| 134 |
('https', 'example.com') |
| 135 |
""" |
| 136 |
schema = schema.lower() |
| 137 |
domain, port = split_domain_port(netloc) |
| 138 |
if (port == "80" and schema in ("http", "")) or (port == "443" and schema in ("https", "")): |
| 139 |
return "http" if port == "80" else "https", domain |
| 140 |
return schema, netloc |
| 141 |
|
| 142 |
|
| 143 |
def hierarchical_str(pattern: str) -> str: |
| 144 |
""" |
| 145 |
Rewrites the given pattern in a string that is useful to sort patterns from more general to more concrete. |
| 146 |
For example, the pattern "example.com" is more general than "blog.example.com" which is more general than |
| 147 |
"blog.example.com/post/1" |
| 148 |
|
| 149 |
>>> hierarchical_str("http://blog.example.com/path?query=23#fragment") |
| 150 |
'com.example.blog/pathquery=23fragment' |
| 151 |
>>> hierarchical_str("http://blog.example.com:1234") |
| 152 |
'com.example.blog' |
| 153 |
>>> hierarchical_str("http://127.0.0.1:80/path") |
| 154 |
'127.0.0.1/path' |
| 155 |
""" |
| 156 |
parsed = pattern_parse(pattern) |
| 157 |
netloc = parsed.netloc |
| 158 |
if ":" in parsed.netloc: |
| 159 |
netloc, _ = split_domain_port(parsed.netloc) |
| 160 |
try: |
| 161 |
ipaddress.ip_address(netloc) |
| 162 |
is_ip = True |
| 163 |
except ValueError: |
| 164 |
is_ip = False |
| 165 |
if not is_ip: |
| 166 |
# Reversing the domain so that higher levels are before |
| 167 |
# e.g. blog.example.com -> com.example.blog |
| 168 |
netloc = ".".join(reversed(netloc.split("."))) |
| 169 |
return "".join((netloc, *parsed[2:])) |
| 170 |
|
| 171 |
|
| 172 |
def split_domain_port(netloc: str) -> tuple[str, str | None]: |
| 173 |
""" |
| 174 |
Splits the netloc into domain and port. |
| 175 |
|
| 176 |
>>> split_domain_port("example.com") |
| 177 |
('example.com', None) |
| 178 |
>>> split_domain_port("example.com:80") |
| 179 |
('example.com', '80') |
| 180 |
""" |
| 181 |
segments = netloc.split(":") |
| 182 |
if len(segments) > 1: |
| 183 |
return ":".join(segments[:-1]), segments[-1] |
| 184 |
return netloc, None |
| 185 |
|
| 186 |
|
| 187 |
class PatternMatcher: |
| 188 |
def __init__(self, pattern: str): |
| 189 |
# Parsing and validation |
| 190 |
self.pattern = pattern |
| 191 |
self.parsed = pattern_parse(pattern) |
| 192 |
self.domain = get_pattern_domain(pattern) |
| 193 |
self.netloc_re: Pattern[str] | None = None |
| 194 |
self.path_re: Pattern[str] | None = None |
| 195 |
self.fragment_re: Pattern[str] | None = None |
| 196 |
self.query_re_dict: dict[str, Pattern[str]] | None = None |
| 197 |
self._build_regexes() |
| 198 |
|
| 199 |
def _build_regexes(self) -> None: |
| 200 |
""" |
| 201 |
Builds the compiled regexes that can be used to match the pattern. |
| 202 |
""" |
| 203 |
pscheme, pnetloc, ppath, pquery, pfragment = self.parsed |
| 204 |
if pnetloc: |
| 205 |
netloc_re = re.escape(pnetloc) |
| 206 |
if not any((ppath, pquery, pfragment)): |
| 207 |
# Also match subdomains if there is no path, query or fragment in the pattern |
| 208 |
netloc_re = rf"(?:.*\.)?{netloc_re}" |
| 209 |
netloc_re = f"^(?:www.)?{netloc_re}$" |
| 210 |
self.netloc_re = re.compile(netloc_re, re.IGNORECASE) |
| 211 |
if ppath: |
| 212 |
self.path_re = self._path_or_fragment_re(ppath) |
| 213 |
if pfragment: |
| 214 |
self.fragment_re = self._path_or_fragment_re(pfragment) |
| 215 |
if pquery: |
| 216 |
pkvs = parse_qs(pquery, keep_blank_values=True) |
| 217 |
query_re_dict = {} |
| 218 |
for pparam, values in pkvs.items(): |
| 219 |
pparam = pparam.lower() # noqa: PLW2901 |
| 220 |
if "*" in pparam: |
| 221 |
warnings.warn( |
| 222 |
f"Wildcard expansion is only allowed for the values in the query parameter. Pattern: '{self.pattern}'", |
| 223 |
SyntaxWarning, |
| 224 |
stacklevel=3, |
| 225 |
) |
| 226 |
pparam = pparam.replace("*", "") # noqa: PLW2901 |
| 227 |
if not pparam: |
| 228 |
continue |
| 229 |
param_re = rf"^(?:{'|'.join([_wildcard_re_escape(value) for value in values])})$" |
| 230 |
query_re_dict[pparam] = re.compile(param_re, re.IGNORECASE) |
| 231 |
self.query_re_dict = query_re_dict or None |
| 232 |
|
| 233 |
def match(self, url: str) -> bool: |
| 234 |
""" |
| 235 |
Return True if the url matches the pattern. |
| 236 |
""" |
| 237 |
parsed = _urlparse(url) |
| 238 |
if self.parsed.scheme and parsed.scheme != self.parsed.scheme: |
| 239 |
return False |
| 240 |
if self.netloc_re and not self.netloc_re.match(parsed.netloc): |
| 241 |
return False |
| 242 |
if self.path_re and not self.path_re.match(parsed.path): |
| 243 |
return False |
| 244 |
if self.fragment_re and not self.fragment_re.match(parsed.fragment): |
| 245 |
return False |
| 246 |
if self.query_re_dict: |
| 247 |
kvs = parse_qs(parsed.query, keep_blank_values=True) |
| 248 |
kvs = {k.lower(): v for k, v in kvs.items()} |
| 249 |
# All params must be present in the URL |
| 250 |
for param, param_re in self.query_re_dict.items(): |
| 251 |
if param not in kvs: |
| 252 |
return False |
| 253 |
if not any(param_re.match(value) for value in (kvs[param])): |
| 254 |
return False |
| 255 |
return True |
| 256 |
|
| 257 |
@staticmethod |
| 258 |
def _path_or_fragment_re(path_or_fragment: str) -> Pattern[str]: |
| 259 |
"""Wildcard expansion + end of line character""" |
| 260 |
re_str = _wildcard_re_escape(path_or_fragment) |
| 261 |
if re_str.endswith(r"\|"): |
| 262 |
# case where the match must be exact |
| 263 |
re_str = re_str[:-2] |
| 264 |
else: |
| 265 |
re_str += r".*" |
| 266 |
re_str = rf"^{re_str}$" |
| 267 |
return re.compile(re_str, re.IGNORECASE) |