| 1 |
""" robotparser.py |
| 2 |
|
| 3 |
Copyright (C) 2000 Bastian Kleineidam |
| 4 |
|
| 5 |
You can choose between two licenses when using this package: |
| 6 |
1) GNU GPLv2 |
| 7 |
2) PSF license for Python 2.2 |
| 8 |
|
| 9 |
The robots.txt Exclusion Protocol is implemented as specified in |
| 10 |
RFC 9309 |
| 11 |
""" |
| 12 |
|
| 13 |
import collections |
| 14 |
import re |
| 15 |
import urllib.error |
| 16 |
import urllib.parse |
| 17 |
import urllib.request |
| 18 |
|
| 19 |
__all__ = ["RobotFileParser"] |
| 20 |
|
| 21 |
RequestRate = collections.namedtuple("RequestRate", "requests seconds") |
| 22 |
|
| 23 |
|
| 24 |
class RobotFileParser: |
| 25 |
""" This class provides a set of methods to read, parse and answer |
| 26 |
questions about a single robots.txt file. |
| 27 |
|
| 28 |
""" |
| 29 |
|
| 30 |
def __init__(self, url=''): |
| 31 |
self.entries = [] |
| 32 |
self.groups = {} |
| 33 |
self.sitemaps = [] |
| 34 |
self.default_entry = None |
| 35 |
self.disallow_all = False |
| 36 |
self.allow_all = False |
| 37 |
self.set_url(url) |
| 38 |
self.last_checked = 0 |
| 39 |
|
| 40 |
def mtime(self): |
| 41 |
"""Returns the time the robots.txt file was last fetched. |
| 42 |
|
| 43 |
This is useful for long-running web spiders that need to |
| 44 |
check for new robots.txt files periodically. |
| 45 |
|
| 46 |
""" |
| 47 |
return self.last_checked |
| 48 |
|
| 49 |
def modified(self): |
| 50 |
"""Sets the time the robots.txt file was last fetched to the |
| 51 |
current time. |
| 52 |
|
| 53 |
""" |
| 54 |
import time |
| 55 |
self.last_checked = time.time() |
| 56 |
|
| 57 |
def set_url(self, url): |
| 58 |
"""Sets the URL referring to a robots.txt file.""" |
| 59 |
self.url = url |
| 60 |
self.host, self.path = urllib.parse.urlsplit(url)[1:3] |
| 61 |
|
| 62 |
def read(self): |
| 63 |
"""Reads the robots.txt URL and feeds it to the parser.""" |
| 64 |
try: |
| 65 |
f = urllib.request.urlopen(self.url) |
| 66 |
except urllib.error.HTTPError as err: |
| 67 |
if err.code in (401, 403): |
| 68 |
self.disallow_all = True |
| 69 |
elif err.code >= 400 and err.code < 500: |
| 70 |
self.allow_all = True |
| 71 |
err.close() |
| 72 |
else: |
| 73 |
raw = f.read() |
| 74 |
self.parse(raw.decode("utf-8", "surrogateescape").splitlines()) |
| 75 |
|
| 76 |
def _add_entry(self, entry): |
| 77 |
self.entries.append(entry) |
| 78 |
for agent in entry.useragents: |
| 79 |
agent = agent.lower() |
| 80 |
if agent not in self.groups: |
| 81 |
self.groups[agent] = entry |
| 82 |
else: |
| 83 |
self.groups[agent] = merge_entries(self.groups[agent], entry) |
| 84 |
|
| 85 |
def parse(self, lines): |
| 86 |
"""Parse the input lines from a robots.txt file. |
| 87 |
|
| 88 |
We allow that a user-agent: line is not preceded by |
| 89 |
one or more blank lines. |
| 90 |
""" |
| 91 |
entries = [] |
| 92 |
# states: |
| 93 |
# 0: start state |
| 94 |
# 1: saw user-agent line |
| 95 |
# 2: saw an allow or disallow line |
| 96 |
state = 0 |
| 97 |
entry = Entry() |
| 98 |
|
| 99 |
self.modified() |
| 100 |
for line in lines: |
| 101 |
# remove optional comment and strip line |
| 102 |
i = line.find('#') |
| 103 |
if i >= 0: |
| 104 |
line = line[:i] |
| 105 |
line = line.strip() |
| 106 |
if not line: |
| 107 |
continue |
| 108 |
line = line.split(':', 1) |
| 109 |
if len(line) == 2: |
| 110 |
line[0] = line[0].strip().lower() |
| 111 |
line[1] = line[1].strip() |
| 112 |
if line[0] == "user-agent": |
| 113 |
if state == 2: |
| 114 |
self._add_entry(entry) |
| 115 |
entry = Entry() |
| 116 |
product_token = line[1] |
| 117 |
entry.useragents.append(product_token) |
| 118 |
state = 1 |
| 119 |
elif line[0] == "disallow": |
| 120 |
if state != 0: |
| 121 |
state = 2 |
| 122 |
try: |
| 123 |
entry.rulelines.append(RuleLine(line[1], False)) |
| 124 |
except ValueError: |
| 125 |
pass |
| 126 |
elif line[0] == "allow": |
| 127 |
if state != 0: |
| 128 |
state = 2 |
| 129 |
try: |
| 130 |
entry.rulelines.append(RuleLine(line[1], True)) |
| 131 |
except ValueError: |
| 132 |
pass |
| 133 |
elif line[0] == "crawl-delay": |
| 134 |
if state != 0: |
| 135 |
# before trying to convert to int we need to make |
| 136 |
# sure that robots.txt has valid syntax otherwise |
| 137 |
# it will crash |
| 138 |
if line[1].strip().isdigit(): |
| 139 |
entry.delay = int(line[1]) |
| 140 |
state = 2 |
| 141 |
elif line[0] == "request-rate": |
| 142 |
if state != 0: |
| 143 |
numbers = line[1].split('/') |
| 144 |
# check if all values are sane |
| 145 |
if (len(numbers) == 2 and numbers[0].strip().isdigit() |
| 146 |
and numbers[1].strip().isdigit()): |
| 147 |
entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1])) |
| 148 |
state = 2 |
| 149 |
elif line[0] == "sitemap": |
| 150 |
# According to http://www.sitemaps.org/protocol.html |
| 151 |
# "This directive is independent of the user-agent line, |
| 152 |
# so it doesn't matter where you place it in your file." |
| 153 |
# Therefore we do not change the state of the parser. |
| 154 |
self.sitemaps.append(line[1]) |
| 155 |
if state != 0: |
| 156 |
self._add_entry(entry) |
| 157 |
|
| 158 |
def _find_entry(self, useragent): |
| 159 |
entry = self.groups.get(useragent.lower()) |
| 160 |
if entry is not None: |
| 161 |
return entry |
| 162 |
for entry in self.groups.values(): |
| 163 |
if entry.applies_to(useragent): |
| 164 |
return entry |
| 165 |
return self.groups.get('*') |
| 166 |
|
| 167 |
def can_fetch(self, useragent, url): |
| 168 |
"""using the parsed robots.txt decide if useragent can fetch url""" |
| 169 |
if self.disallow_all: |
| 170 |
return False |
| 171 |
if self.allow_all: |
| 172 |
return True |
| 173 |
# Until the robots.txt file has been read or found not |
| 174 |
# to exist, we must assume that no url is allowable. |
| 175 |
# This prevents false positives when a user erroneously |
| 176 |
# calls can_fetch() before calling read(). |
| 177 |
if not self.last_checked: |
| 178 |
return False |
| 179 |
# TODO: The private API is used in order to preserve an empty query. |
| 180 |
# This is temporary until the public API starts supporting this feature. |
| 181 |
parsed_url = urllib.parse._urlsplit(url, '') |
| 182 |
url = urllib.parse._urlunsplit(None, None, *parsed_url[2:]) |
| 183 |
url = normalize_uri(url) |
| 184 |
if not url: |
| 185 |
url = "/" |
| 186 |
if url == '/robots.txt': |
| 187 |
# The /robots.txt URI is implicitly allowed. |
| 188 |
return True |
| 189 |
entry = self._find_entry(useragent) |
| 190 |
if entry is None: |
| 191 |
return True |
| 192 |
return entry.allowance(url) |
| 193 |
|
| 194 |
def crawl_delay(self, useragent): |
| 195 |
if not self.mtime(): |
| 196 |
return None |
| 197 |
entry = self._find_entry(useragent) |
| 198 |
if entry is None: |
| 199 |
return None |
| 200 |
return entry.delay |
| 201 |
|
| 202 |
def request_rate(self, useragent): |
| 203 |
if not self.mtime(): |
| 204 |
return None |
| 205 |
entry = self._find_entry(useragent) |
| 206 |
if entry is None: |
| 207 |
return None |
| 208 |
return entry.req_rate |
| 209 |
|
| 210 |
def site_maps(self): |
| 211 |
if not self.sitemaps: |
| 212 |
return None |
| 213 |
return self.sitemaps |
| 214 |
|
| 215 |
def __str__(self): |
| 216 |
entries = self.entries |
| 217 |
if self.default_entry is not None: |
| 218 |
entries = entries + [self.default_entry] |
| 219 |
return '\n\n'.join(filter(None, map(str, entries))) |
| 220 |
|
| 221 |
class RuleLine: |
| 222 |
"""A rule line is a single "Allow:" (allowance==True) or "Disallow:" |
| 223 |
(allowance==False) followed by a path.""" |
| 224 |
def __init__(self, path, allowance): |
| 225 |
if path == '' and not allowance: |
| 226 |
# an empty value means allow all |
| 227 |
allowance = True |
| 228 |
path = re.sub(r'[*]{2,}', '*', path) |
| 229 |
path = re.sub(r'[$][$*]+', '$', path) |
| 230 |
path = normalize_pattern(path) |
| 231 |
self.fullmatch = path.endswith('$') |
| 232 |
path = path.rstrip('$') |
| 233 |
if '$' in path: |
| 234 |
raise ValueError('$ not at the end of path') |
| 235 |
self.matcher = None |
| 236 |
if '*' in path: |
| 237 |
pattern = re.compile(translate_pattern(path), re.DOTALL) |
| 238 |
if self.fullmatch: |
| 239 |
self.matcher = pattern.fullmatch |
| 240 |
else: |
| 241 |
self.matcher = pattern.match |
| 242 |
self.path = path |
| 243 |
self.allowance = allowance |
| 244 |
|
| 245 |
def applies_to(self, filename): |
| 246 |
# If the filename matches the rule, return the matching length plus 1. |
| 247 |
# If it does not match, return 0. |
| 248 |
if self.matcher is not None: |
| 249 |
m = self.matcher(filename) |
| 250 |
if m: |
| 251 |
return m.end() + 1 |
| 252 |
else: |
| 253 |
if self.fullmatch: |
| 254 |
if filename == self.path: |
| 255 |
return len(self.path) + 1 |
| 256 |
else: |
| 257 |
if filename.startswith(self.path): |
| 258 |
return len(self.path) + 1 |
| 259 |
return 0 |
| 260 |
|
| 261 |
def __str__(self): |
| 262 |
return (("Allow" if self.allowance else "Disallow") + ": " + self.path |
| 263 |
+ ('$' if self.fullmatch else '')) |
| 264 |
|
| 265 |
|
| 266 |
class Entry: |
| 267 |
"""An entry has one or more user-agents and zero or more rulelines""" |
| 268 |
def __init__(self): |
| 269 |
self.useragents = [] |
| 270 |
self.rulelines = [] |
| 271 |
self.delay = None |
| 272 |
self.req_rate = None |
| 273 |
|
| 274 |
def __str__(self): |
| 275 |
if not self.useragents: |
| 276 |
return '' |
| 277 |
ret = [] |
| 278 |
for agent in self.useragents: |
| 279 |
ret.append(f"User-agent: {agent}") |
| 280 |
if self.delay is not None: |
| 281 |
ret.append(f"Crawl-delay: {self.delay}") |
| 282 |
if self.req_rate is not None: |
| 283 |
rate = self.req_rate |
| 284 |
ret.append(f"Request-rate: {rate.requests}/{rate.seconds}") |
| 285 |
if self.rulelines: |
| 286 |
ret.extend(map(str, self.rulelines)) |
| 287 |
else: |
| 288 |
ret.append("Allow:") |
| 289 |
return '\n'.join(ret) |
| 290 |
|
| 291 |
def applies_to(self, useragent): |
| 292 |
"""check if this entry applies to the specified agent""" |
| 293 |
if useragent is None: |
| 294 |
return '*' in self.useragents |
| 295 |
# split the name token and make it lower case |
| 296 |
useragent = useragent.split("/")[0].lower() |
| 297 |
for agent in self.useragents: |
| 298 |
if agent != '*': |
| 299 |
agent = agent.lower() |
| 300 |
if agent in useragent: |
| 301 |
return True |
| 302 |
return False |
| 303 |
|
| 304 |
def allowance(self, filename): |
| 305 |
"""Preconditions: |
| 306 |
- our agent applies to this entry |
| 307 |
- filename is URL encoded |
| 308 |
""" |
| 309 |
best_match = -1 |
| 310 |
allowance = True |
| 311 |
for line in self.rulelines: |
| 312 |
m = line.applies_to(filename) |
| 313 |
if m: |
| 314 |
if m > best_match: |
| 315 |
best_match = m |
| 316 |
allowance = line.allowance |
| 317 |
elif m == best_match and not allowance: |
| 318 |
allowance = line.allowance |
| 319 |
return allowance |
| 320 |
|
| 321 |
|
| 322 |
def normalize(path): |
| 323 |
unquoted = urllib.parse.unquote(path, errors='surrogateescape') |
| 324 |
return urllib.parse.quote(unquoted, errors='surrogateescape') |
| 325 |
|
| 326 |
def normalize_uri(path): |
| 327 |
path, sep, query = path.partition('?') |
| 328 |
path = normalize(path) |
| 329 |
if sep: |
| 330 |
query = re.sub(r'[^=&]+', lambda m: normalize(m[0]), query) |
| 331 |
path += '?' + query |
| 332 |
return path |
| 333 |
|
| 334 |
def normalize_pattern(path): |
| 335 |
path, sep, query = path.partition('?') |
| 336 |
path = re.sub(r'[^*$]+', lambda m: normalize(m[0]), path) |
| 337 |
if sep: |
| 338 |
query = re.sub(r'[^=&*$]+', lambda m: normalize(m[0]), query) |
| 339 |
path += '?' + query |
| 340 |
return path |
| 341 |
|
| 342 |
def translate_pattern(path): |
| 343 |
parts = list(map(re.escape, path.split('*'))) |
| 344 |
for i in range(1, len(parts)-1): |
| 345 |
parts[i] = f'(?>.*?{parts[i]})' |
| 346 |
parts[-1] = f'.*{parts[-1]}' |
| 347 |
return ''.join(parts) |
| 348 |
|
| 349 |
def merge_entries(e1, e2): |
| 350 |
entry = Entry() |
| 351 |
entry.useragents = list(filter(set(e2.useragents).__contains__, e1.useragents)) |
| 352 |
entry.rulelines = e1.rulelines + e2.rulelines |
| 353 |
entry.delay = e1.delay if e2.delay is None else e2.delay |
| 354 |
entry.req_rate = e1.req_rate if e2.req_rate is None else e2.req_rate |
| 355 |
return entry |