Debian Codegraph @

url_matcher/patterns.py

raw
1 """
2 Utilities to parse patterns and match URLs using them.
3 """
4
5 from __future__ import annotations
6
7 import ipaddress
8 import re
9 import warnings
10 from functools import lru_cache
11 from re import Pattern
12 from typing import NamedTuple
13 from urllib.parse import parse_qs, urlparse
14
15 from url_matcher.util import get_domain
16
17
18 def get_pattern_domain(pattern: str) -> str | None:
19 """
20 Returns the domain of the pattern if any.
21
22 >>> get_pattern_domain("")
23
24 >>> get_pattern_domain("/")
25
26 >>> get_pattern_domain("dom")
27 'dom'
28 >>> get_pattern_domain("DOM")
29 'dom'
30 >>> get_pattern_domain("dom:80")
31 'dom'
32 >>> get_pattern_domain("http://dom:80")
33 'dom'
34 >>> get_pattern_domain("http://dom/a_path")
35 'dom'
36 """
37 parsed = pattern_parse(pattern)
38 if parsed.netloc:
39 return get_domain(f"//{parsed.netloc}").lower()
40 return None
41
42
43 def pattern_to_url(pattern: str) -> str:
44 """
45 Required for urlparse to recognize the domain in patterns
46 like example.com/path
47
48 >>> pattern_to_url("example.com/")
49 '//example.com/'
50 >>> pattern_to_url("example.com")
51 '//example.com'
52 >>> pattern_to_url("https://example.com")
53 'https://example.com'
54 >>> pattern_to_url("MySchema4+.-://example.com")
55 'MySchema4+.-://example.com'
56 >>> pattern_to_url("//example.com")
57 '////example.com'
58 """
59 # As defined in https://datatracker.ietf.org/doc/html/rfc3986#section-3.1
60 has_scheme = re.search(r"^([a-zA-Z][a-zA-Z0-9.+-]*:)?//", pattern)
61 if not has_scheme:
62 pattern = f"//{pattern}"
63 elif pattern.startswith("//"):
64 # This is required because urlparse("//example.com").netloc == "//example.com"
65 # but instead we want it to be parsed into the the path. We achieve it by appending
66 # two more slashes
67 pattern = f"//{pattern}"
68 return pattern
69
70
71 class ParseTuple(NamedTuple):
72 scheme: str
73 netloc: str
74 path: str
75 query: str
76 fragment: str
77
78
79 @lru_cache(30)
80 def pattern_parse(pattern: str) -> ParseTuple:
81 """
82 Parses the pattern to a named tuple (scheme, netloc, path, query, fragment)
83 >>> pattern_parse("example.com")
84 ParseTuple(scheme='', netloc='example.com', path='', query='', fragment='')
85 >>> pattern_parse("//example.com/path;this_is_also_path")
86 ParseTuple(scheme='', netloc='', path='//example.com/path;this_is_also_path', query='', fragment='')
87 """
88 pattern = pattern_to_url(pattern)
89 return _urlparse(pattern)
90
91
92 def _urlparse(url: str) -> ParseTuple:
93 """
94 Returns a named tuple (scheme, netloc, path, query, fragment)
95 where path and params are joined together into path and
96 some other elements are normalized.
97
98 >>> _urlparse("scheme://example.com/path;params?query=23#fragment")
99 ParseTuple(scheme='scheme', netloc='example.com', path='/path;params', query='query=23', fragment='fragment')
100 >>> _urlparse("http://example.com:80/path")
101 ParseTuple(scheme='http', netloc='example.com', path='/path', query='', fragment='')
102 """
103 scheme, netloc, path, params, query, fragment = urlparse(url)
104 path = _join_path_and_params(path, params)
105 scheme, netloc = normalize_netloc_and_schema(scheme, netloc)
106 return ParseTuple(scheme, netloc, path, query, fragment)
107
108
109 def _wildcard_re_escape(text: str) -> str:
110 return re.escape(text).replace("\\*", ".*")
111
112
113 def _join_path_and_params(path: str, params: str) -> str:
114 if params:
115 return f"{path};{params}"
116 return path
117
118
119 def normalize_netloc_and_schema(schema: str, netloc: str) -> tuple[str, str]:
120 """
121 Removes 80 or 443 port when obvious. Deduces http or https when the port is provided
122
123 >>> normalize_netloc_and_schema("http", "example.com:80")
124 ('http', 'example.com')
125 >>> normalize_netloc_and_schema("http", "example.com:80")
126 ('http', 'example.com')
127 >>> normalize_netloc_and_schema("http", "example.com:443")
128 ('http', 'example.com:443')
129 >>> normalize_netloc_and_schema("https", "example.com:443")
130 ('https', 'example.com')
131 >>> normalize_netloc_and_schema("", "example.com:80")
132 ('http', 'example.com')
133 >>> normalize_netloc_and_schema("", "example.com:443")
134 ('https', 'example.com')
135 """
136 schema = schema.lower()
137 domain, port = split_domain_port(netloc)
138 if (port == "80" and schema in ("http", "")) or (port == "443" and schema in ("https", "")):
139 return "http" if port == "80" else "https", domain
140 return schema, netloc
141
142
143 def hierarchical_str(pattern: str) -> str:
144 """
145 Rewrites the given pattern in a string that is useful to sort patterns from more general to more concrete.
146 For example, the pattern "example.com" is more general than "blog.example.com" which is more general than
147 "blog.example.com/post/1"
148
149 >>> hierarchical_str("http://blog.example.com/path?query=23#fragment")
150 'com.example.blog/pathquery=23fragment'
151 >>> hierarchical_str("http://blog.example.com:1234")
152 'com.example.blog'
153 >>> hierarchical_str("http://127.0.0.1:80/path")
154 '127.0.0.1/path'
155 """
156 parsed = pattern_parse(pattern)
157 netloc = parsed.netloc
158 if ":" in parsed.netloc:
159 netloc, _ = split_domain_port(parsed.netloc)
160 try:
161 ipaddress.ip_address(netloc)
162 is_ip = True
163 except ValueError:
164 is_ip = False
165 if not is_ip:
166 # Reversing the domain so that higher levels are before
167 # e.g. blog.example.com -> com.example.blog
168 netloc = ".".join(reversed(netloc.split(".")))
169 return "".join((netloc, *parsed[2:]))
170
171
172 def split_domain_port(netloc: str) -> tuple[str, str | None]:
173 """
174 Splits the netloc into domain and port.
175
176 >>> split_domain_port("example.com")
177 ('example.com', None)
178 >>> split_domain_port("example.com:80")
179 ('example.com', '80')
180 """
181 segments = netloc.split(":")
182 if len(segments) > 1:
183 return ":".join(segments[:-1]), segments[-1]
184 return netloc, None
185
186
187 class PatternMatcher:
188 def __init__(self, pattern: str):
189 # Parsing and validation
190 self.pattern = pattern
191 self.parsed = pattern_parse(pattern)
192 self.domain = get_pattern_domain(pattern)
193 self.netloc_re: Pattern[str] | None = None
194 self.path_re: Pattern[str] | None = None
195 self.fragment_re: Pattern[str] | None = None
196 self.query_re_dict: dict[str, Pattern[str]] | None = None
197 self._build_regexes()
198
199 def _build_regexes(self) -> None:
200 """
201 Builds the compiled regexes that can be used to match the pattern.
202 """
203 pscheme, pnetloc, ppath, pquery, pfragment = self.parsed
204 if pnetloc:
205 netloc_re = re.escape(pnetloc)
206 if not any((ppath, pquery, pfragment)):
207 # Also match subdomains if there is no path, query or fragment in the pattern
208 netloc_re = rf"(?:.*\.)?{netloc_re}"
209 netloc_re = f"^(?:www.)?{netloc_re}$"
210 self.netloc_re = re.compile(netloc_re, re.IGNORECASE)
211 if ppath:
212 self.path_re = self._path_or_fragment_re(ppath)
213 if pfragment:
214 self.fragment_re = self._path_or_fragment_re(pfragment)
215 if pquery:
216 pkvs = parse_qs(pquery, keep_blank_values=True)
217 query_re_dict = {}
218 for pparam, values in pkvs.items():
219 pparam = pparam.lower() # noqa: PLW2901
220 if "*" in pparam:
221 warnings.warn(
222 f"Wildcard expansion is only allowed for the values in the query parameter. Pattern: '{self.pattern}'",
223 SyntaxWarning,
224 stacklevel=3,
225 )
226 pparam = pparam.replace("*", "") # noqa: PLW2901
227 if not pparam:
228 continue
229 param_re = rf"^(?:{'|'.join([_wildcard_re_escape(value) for value in values])})$"
230 query_re_dict[pparam] = re.compile(param_re, re.IGNORECASE)
231 self.query_re_dict = query_re_dict or None
232
233 def match(self, url: str) -> bool:
234 """
235 Return True if the url matches the pattern.
236 """
237 parsed = _urlparse(url)
238 if self.parsed.scheme and parsed.scheme != self.parsed.scheme:
239 return False
240 if self.netloc_re and not self.netloc_re.match(parsed.netloc):
241 return False
242 if self.path_re and not self.path_re.match(parsed.path):
243 return False
244 if self.fragment_re and not self.fragment_re.match(parsed.fragment):
245 return False
246 if self.query_re_dict:
247 kvs = parse_qs(parsed.query, keep_blank_values=True)
248 kvs = {k.lower(): v for k, v in kvs.items()}
249 # All params must be present in the URL
250 for param, param_re in self.query_re_dict.items():
251 if param not in kvs:
252 return False
253 if not any(param_re.match(value) for value in (kvs[param])):
254 return False
255 return True
256
257 @staticmethod
258 def _path_or_fragment_re(path_or_fragment: str) -> Pattern[str]:
259 """Wildcard expansion + end of line character"""
260 re_str = _wildcard_re_escape(path_or_fragment)
261 if re_str.endswith(r"\|"):
262 # case where the match must be exact
263 re_str = re_str[:-2]
264 else:
265 re_str += r".*"
266 re_str = rf"^{re_str}$"
267 return re.compile(re_str, re.IGNORECASE)