| 1 |
""" |
| 2 |
Example of usage of the URLMatcher library |
| 3 |
""" |
| 4 |
|
| 5 |
from __future__ import annotations |
| 6 |
|
| 7 |
import dataclasses |
| 8 |
import random |
| 9 |
import time |
| 10 |
from collections import Counter |
| 11 |
|
| 12 |
from url_matcher import Patterns, URLMatcher |
| 13 |
from url_matcher.matcher import IncludePatternsWithoutDomainError |
| 14 |
|
| 15 |
matcher = URLMatcher() |
| 16 |
|
| 17 |
# Let's add a rule for books to scrape product |
| 18 |
patterns = Patterns(include=["books.toscrape.com/catalogue/"], exclude=["/catalogue/category/"]) |
| 19 |
matcher.add_or_update("books product", patterns) |
| 20 |
|
| 21 |
# Now a rule for product list in books to scrape |
| 22 |
patterns = Patterns( |
| 23 |
include=["books.toscrape.com/catalogue/category/", "books.toscrape.com/|", "books.toscrape.com/index.html|"] |
| 24 |
) |
| 25 |
matcher.add_or_update("books productList", patterns) |
| 26 |
|
| 27 |
|
| 28 |
# Let's try it |
| 29 |
|
| 30 |
url = "https://books.toscrape.com/catalogue/soumission_998/index.html" |
| 31 |
assert matcher.match(url) == "books product" |
| 32 |
|
| 33 |
url = "https://books.toscrape.com/catalogue/category/books/fiction_10/index.html" |
| 34 |
assert matcher.match(url) == "books productList" |
| 35 |
|
| 36 |
url = "https://amazon.com" |
| 37 |
assert not matcher.match(url) |
| 38 |
|
| 39 |
# Adding a pattern without domain fails |
| 40 |
|
| 41 |
try: |
| 42 |
matcher.add_or_update("won't work", Patterns(["/path"])) |
| 43 |
raise AssertionError |
| 44 |
except IncludePatternsWithoutDomainError: |
| 45 |
... |
| 46 |
|
| 47 |
# But the empty pattern works. It matches anything |
| 48 |
|
| 49 |
assert URLMatcher({"Anything": Patterns([""])}).match("http://anything") |
| 50 |
|
| 51 |
# Now let's see that priorities are working. They are applied only if several |
| 52 |
# rules match the URL. |
| 53 |
|
| 54 |
patterns = Patterns(["priority.com"]) |
| 55 |
matcher.add_or_update("low priority", dataclasses.replace(patterns, priority=200)) |
| 56 |
matcher.add_or_update("high priority", dataclasses.replace(patterns, priority=300)) |
| 57 |
assert matcher.match("http://priority.com") == "high priority" |
| 58 |
|
| 59 |
# Let's invert the priorities |
| 60 |
|
| 61 |
matcher.add_or_update("low priority", dataclasses.replace(patterns, priority=300)) |
| 62 |
matcher.add_or_update("high priority", dataclasses.replace(patterns, priority=200)) |
| 63 |
assert matcher.match("http://priority.com") == "low priority" |
| 64 |
|
| 65 |
# Let's check the speed creating patterns for many domains and matching |
| 66 |
# urls for these domains. |
| 67 |
|
| 68 |
|
| 69 |
def add_patterns(domain: int) -> None: |
| 70 |
patterns = Patterns(include=[f"{domain}/catalogue/?param=book"], exclude=["/catalogue/category/"]) |
| 71 |
matcher.add_or_update(f"{domain} product", patterns) |
| 72 |
|
| 73 |
patterns = Patterns(include=[f"{domain}/catalogue/category/?param=book_list", f"{domain}/", f"{domain}/index.html"]) |
| 74 |
matcher.add_or_update(f"{domain} productList", patterns) |
| 75 |
|
| 76 |
|
| 77 |
N_DOMAINS = 500 |
| 78 |
N_URLS = 300 |
| 79 |
|
| 80 |
URLS = [ |
| 81 |
"https://books.toscrape.com/catalogue/soumission_998/index.html?param=book&p1=23&p2=45", |
| 82 |
"https://books.toscrape.com/catalogue/category/books/fiction_10/index.html?param=book_list&p5=23&p6=45", |
| 83 |
] |
| 84 |
|
| 85 |
# Adding the patterns |
| 86 |
for idx in range(N_DOMAINS): |
| 87 |
add_patterns(idx) |
| 88 |
|
| 89 |
urls = [] |
| 90 |
for _ in range(N_URLS): |
| 91 |
url = random.choice(URLS) |
| 92 |
domain = random.randint(0, N_DOMAINS - 1) |
| 93 |
url = url.replace("books.toscrape.com", f"{domain}") |
| 94 |
urls.append((domain, url)) |
| 95 |
|
| 96 |
# Let's try to match the urls |
| 97 |
start = time.perf_counter() |
| 98 |
counter: Counter[bool] = Counter() |
| 99 |
for domain, url in urls: |
| 100 |
match = matcher.match(url) |
| 101 |
counter[bool(match)] += 1 |
| 102 |
assert match and f"{domain}" in match |
| 103 |
end = time.perf_counter() |
| 104 |
|
| 105 |
# It took in my machine ~ 0.04 millis per URL |
| 106 |
print(f"{((end - start) / N_URLS) * 1000:.3f} milliseconds per URL. Total {end - start} seconds to match {N_URLS} URLs") |
| 107 |
|
| 108 |
print("Everything worked fine!") |