Debian Codegraph @

url_matcher/example.py

raw
1 """
2 Example of usage of the URLMatcher library
3 """
4
5 from __future__ import annotations
6
7 import dataclasses
8 import random
9 import time
10 from collections import Counter
11
12 from url_matcher import Patterns, URLMatcher
13 from url_matcher.matcher import IncludePatternsWithoutDomainError
14
15 matcher = URLMatcher()
16
17 # Let's add a rule for books to scrape product
18 patterns = Patterns(include=["books.toscrape.com/catalogue/"], exclude=["/catalogue/category/"])
19 matcher.add_or_update("books product", patterns)
20
21 # Now a rule for product list in books to scrape
22 patterns = Patterns(
23 include=["books.toscrape.com/catalogue/category/", "books.toscrape.com/|", "books.toscrape.com/index.html|"]
24 )
25 matcher.add_or_update("books productList", patterns)
26
27
28 # Let's try it
29
30 url = "https://books.toscrape.com/catalogue/soumission_998/index.html"
31 assert matcher.match(url) == "books product"
32
33 url = "https://books.toscrape.com/catalogue/category/books/fiction_10/index.html"
34 assert matcher.match(url) == "books productList"
35
36 url = "https://amazon.com"
37 assert not matcher.match(url)
38
39 # Adding a pattern without domain fails
40
41 try:
42 matcher.add_or_update("won't work", Patterns(["/path"]))
43 raise AssertionError
44 except IncludePatternsWithoutDomainError:
45 ...
46
47 # But the empty pattern works. It matches anything
48
49 assert URLMatcher({"Anything": Patterns([""])}).match("http://anything")
50
51 # Now let's see that priorities are working. They are applied only if several
52 # rules match the URL.
53
54 patterns = Patterns(["priority.com"])
55 matcher.add_or_update("low priority", dataclasses.replace(patterns, priority=200))
56 matcher.add_or_update("high priority", dataclasses.replace(patterns, priority=300))
57 assert matcher.match("http://priority.com") == "high priority"
58
59 # Let's invert the priorities
60
61 matcher.add_or_update("low priority", dataclasses.replace(patterns, priority=300))
62 matcher.add_or_update("high priority", dataclasses.replace(patterns, priority=200))
63 assert matcher.match("http://priority.com") == "low priority"
64
65 # Let's check the speed creating patterns for many domains and matching
66 # urls for these domains.
67
68
69 def add_patterns(domain: int) -> None:
70 patterns = Patterns(include=[f"{domain}/catalogue/?param=book"], exclude=["/catalogue/category/"])
71 matcher.add_or_update(f"{domain} product", patterns)
72
73 patterns = Patterns(include=[f"{domain}/catalogue/category/?param=book_list", f"{domain}/", f"{domain}/index.html"])
74 matcher.add_or_update(f"{domain} productList", patterns)
75
76
77 N_DOMAINS = 500
78 N_URLS = 300
79
80 URLS = [
81 "https://books.toscrape.com/catalogue/soumission_998/index.html?param=book&p1=23&p2=45",
82 "https://books.toscrape.com/catalogue/category/books/fiction_10/index.html?param=book_list&p5=23&p6=45",
83 ]
84
85 # Adding the patterns
86 for idx in range(N_DOMAINS):
87 add_patterns(idx)
88
89 urls = []
90 for _ in range(N_URLS):
91 url = random.choice(URLS)
92 domain = random.randint(0, N_DOMAINS - 1)
93 url = url.replace("books.toscrape.com", f"{domain}")
94 urls.append((domain, url))
95
96 # Let's try to match the urls
97 start = time.perf_counter()
98 counter: Counter[bool] = Counter()
99 for domain, url in urls:
100 match = matcher.match(url)
101 counter[bool(match)] += 1
102 assert match and f"{domain}" in match
103 end = time.perf_counter()
104
105 # It took in my machine ~ 0.04 millis per URL
106 print(f"{((end - start) / N_URLS) * 1000:.3f} milliseconds per URL. Total {end - start} seconds to match {N_URLS} URLs")
107
108 print("Everything worked fine!")