Debian Codegraph @

Lib/urllib/robotparser.py

raw
1 """ robotparser.py
2
3 Copyright (C) 2000 Bastian Kleineidam
4
5 You can choose between two licenses when using this package:
6 1) GNU GPLv2
7 2) PSF license for Python 2.2
8
9 The robots.txt Exclusion Protocol is implemented as specified in
10 RFC 9309
11 """
12
13 import collections
14 import re
15 import urllib.error
16 import urllib.parse
17 import urllib.request
18
19 __all__ = ["RobotFileParser"]
20
21 RequestRate = collections.namedtuple("RequestRate", "requests seconds")
22
23
24 class RobotFileParser:
25 """ This class provides a set of methods to read, parse and answer
26 questions about a single robots.txt file.
27
28 """
29
30 def __init__(self, url=''):
31 self.entries = []
32 self.groups = {}
33 self.sitemaps = []
34 self.default_entry = None
35 self.disallow_all = False
36 self.allow_all = False
37 self.set_url(url)
38 self.last_checked = 0
39
40 def mtime(self):
41 """Returns the time the robots.txt file was last fetched.
42
43 This is useful for long-running web spiders that need to
44 check for new robots.txt files periodically.
45
46 """
47 return self.last_checked
48
49 def modified(self):
50 """Sets the time the robots.txt file was last fetched to the
51 current time.
52
53 """
54 import time
55 self.last_checked = time.time()
56
57 def set_url(self, url):
58 """Sets the URL referring to a robots.txt file."""
59 self.url = url
60 self.host, self.path = urllib.parse.urlsplit(url)[1:3]
61
62 def read(self):
63 """Reads the robots.txt URL and feeds it to the parser."""
64 try:
65 f = urllib.request.urlopen(self.url)
66 except urllib.error.HTTPError as err:
67 if err.code in (401, 403):
68 self.disallow_all = True
69 elif err.code >= 400 and err.code < 500:
70 self.allow_all = True
71 err.close()
72 else:
73 raw = f.read()
74 self.parse(raw.decode("utf-8", "surrogateescape").splitlines())
75
76 def _add_entry(self, entry):
77 self.entries.append(entry)
78 for agent in entry.useragents:
79 agent = agent.lower()
80 if agent not in self.groups:
81 self.groups[agent] = entry
82 else:
83 self.groups[agent] = merge_entries(self.groups[agent], entry)
84
85 def parse(self, lines):
86 """Parse the input lines from a robots.txt file.
87
88 We allow that a user-agent: line is not preceded by
89 one or more blank lines.
90 """
91 entries = []
92 # states:
93 # 0: start state
94 # 1: saw user-agent line
95 # 2: saw an allow or disallow line
96 state = 0
97 entry = Entry()
98
99 self.modified()
100 for line in lines:
101 # remove optional comment and strip line
102 i = line.find('#')
103 if i >= 0:
104 line = line[:i]
105 line = line.strip()
106 if not line:
107 continue
108 line = line.split(':', 1)
109 if len(line) == 2:
110 line[0] = line[0].strip().lower()
111 line[1] = line[1].strip()
112 if line[0] == "user-agent":
113 if state == 2:
114 self._add_entry(entry)
115 entry = Entry()
116 product_token = line[1]
117 entry.useragents.append(product_token)
118 state = 1
119 elif line[0] == "disallow":
120 if state != 0:
121 state = 2
122 try:
123 entry.rulelines.append(RuleLine(line[1], False))
124 except ValueError:
125 pass
126 elif line[0] == "allow":
127 if state != 0:
128 state = 2
129 try:
130 entry.rulelines.append(RuleLine(line[1], True))
131 except ValueError:
132 pass
133 elif line[0] == "crawl-delay":
134 if state != 0:
135 # before trying to convert to int we need to make
136 # sure that robots.txt has valid syntax otherwise
137 # it will crash
138 if line[1].strip().isdigit():
139 entry.delay = int(line[1])
140 state = 2
141 elif line[0] == "request-rate":
142 if state != 0:
143 numbers = line[1].split('/')
144 # check if all values are sane
145 if (len(numbers) == 2 and numbers[0].strip().isdigit()
146 and numbers[1].strip().isdigit()):
147 entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1]))
148 state = 2
149 elif line[0] == "sitemap":
150 # According to http://www.sitemaps.org/protocol.html
151 # "This directive is independent of the user-agent line,
152 # so it doesn't matter where you place it in your file."
153 # Therefore we do not change the state of the parser.
154 self.sitemaps.append(line[1])
155 if state != 0:
156 self._add_entry(entry)
157
158 def _find_entry(self, useragent):
159 entry = self.groups.get(useragent.lower())
160 if entry is not None:
161 return entry
162 for entry in self.groups.values():
163 if entry.applies_to(useragent):
164 return entry
165 return self.groups.get('*')
166
167 def can_fetch(self, useragent, url):
168 """using the parsed robots.txt decide if useragent can fetch url"""
169 if self.disallow_all:
170 return False
171 if self.allow_all:
172 return True
173 # Until the robots.txt file has been read or found not
174 # to exist, we must assume that no url is allowable.
175 # This prevents false positives when a user erroneously
176 # calls can_fetch() before calling read().
177 if not self.last_checked:
178 return False
179 # TODO: The private API is used in order to preserve an empty query.
180 # This is temporary until the public API starts supporting this feature.
181 parsed_url = urllib.parse._urlsplit(url, '')
182 url = urllib.parse._urlunsplit(None, None, *parsed_url[2:])
183 url = normalize_uri(url)
184 if not url:
185 url = "/"
186 if url == '/robots.txt':
187 # The /robots.txt URI is implicitly allowed.
188 return True
189 entry = self._find_entry(useragent)
190 if entry is None:
191 return True
192 return entry.allowance(url)
193
194 def crawl_delay(self, useragent):
195 if not self.mtime():
196 return None
197 entry = self._find_entry(useragent)
198 if entry is None:
199 return None
200 return entry.delay
201
202 def request_rate(self, useragent):
203 if not self.mtime():
204 return None
205 entry = self._find_entry(useragent)
206 if entry is None:
207 return None
208 return entry.req_rate
209
210 def site_maps(self):
211 if not self.sitemaps:
212 return None
213 return self.sitemaps
214
215 def __str__(self):
216 entries = self.entries
217 if self.default_entry is not None:
218 entries = entries + [self.default_entry]
219 return '\n\n'.join(filter(None, map(str, entries)))
220
221 class RuleLine:
222 """A rule line is a single "Allow:" (allowance==True) or "Disallow:"
223 (allowance==False) followed by a path."""
224 def __init__(self, path, allowance):
225 if path == '' and not allowance:
226 # an empty value means allow all
227 allowance = True
228 path = re.sub(r'[*]{2,}', '*', path)
229 path = re.sub(r'[$][$*]+', '$', path)
230 path = normalize_pattern(path)
231 self.fullmatch = path.endswith('$')
232 path = path.rstrip('$')
233 if '$' in path:
234 raise ValueError('$ not at the end of path')
235 self.matcher = None
236 if '*' in path:
237 pattern = re.compile(translate_pattern(path), re.DOTALL)
238 if self.fullmatch:
239 self.matcher = pattern.fullmatch
240 else:
241 self.matcher = pattern.match
242 self.path = path
243 self.allowance = allowance
244
245 def applies_to(self, filename):
246 # If the filename matches the rule, return the matching length plus 1.
247 # If it does not match, return 0.
248 if self.matcher is not None:
249 m = self.matcher(filename)
250 if m:
251 return m.end() + 1
252 else:
253 if self.fullmatch:
254 if filename == self.path:
255 return len(self.path) + 1
256 else:
257 if filename.startswith(self.path):
258 return len(self.path) + 1
259 return 0
260
261 def __str__(self):
262 return (("Allow" if self.allowance else "Disallow") + ": " + self.path
263 + ('$' if self.fullmatch else ''))
264
265
266 class Entry:
267 """An entry has one or more user-agents and zero or more rulelines"""
268 def __init__(self):
269 self.useragents = []
270 self.rulelines = []
271 self.delay = None
272 self.req_rate = None
273
274 def __str__(self):
275 if not self.useragents:
276 return ''
277 ret = []
278 for agent in self.useragents:
279 ret.append(f"User-agent: {agent}")
280 if self.delay is not None:
281 ret.append(f"Crawl-delay: {self.delay}")
282 if self.req_rate is not None:
283 rate = self.req_rate
284 ret.append(f"Request-rate: {rate.requests}/{rate.seconds}")
285 if self.rulelines:
286 ret.extend(map(str, self.rulelines))
287 else:
288 ret.append("Allow:")
289 return '\n'.join(ret)
290
291 def applies_to(self, useragent):
292 """check if this entry applies to the specified agent"""
293 if useragent is None:
294 return '*' in self.useragents
295 # split the name token and make it lower case
296 useragent = useragent.split("/")[0].lower()
297 for agent in self.useragents:
298 if agent != '*':
299 agent = agent.lower()
300 if agent in useragent:
301 return True
302 return False
303
304 def allowance(self, filename):
305 """Preconditions:
306 - our agent applies to this entry
307 - filename is URL encoded
308 """
309 best_match = -1
310 allowance = True
311 for line in self.rulelines:
312 m = line.applies_to(filename)
313 if m:
314 if m > best_match:
315 best_match = m
316 allowance = line.allowance
317 elif m == best_match and not allowance:
318 allowance = line.allowance
319 return allowance
320
321
322 def normalize(path):
323 unquoted = urllib.parse.unquote(path, errors='surrogateescape')
324 return urllib.parse.quote(unquoted, errors='surrogateescape')
325
326 def normalize_uri(path):
327 path, sep, query = path.partition('?')
328 path = normalize(path)
329 if sep:
330 query = re.sub(r'[^=&]+', lambda m: normalize(m[0]), query)
331 path += '?' + query
332 return path
333
334 def normalize_pattern(path):
335 path, sep, query = path.partition('?')
336 path = re.sub(r'[^*$]+', lambda m: normalize(m[0]), path)
337 if sep:
338 query = re.sub(r'[^=&*$]+', lambda m: normalize(m[0]), query)
339 path += '?' + query
340 return path
341
342 def translate_pattern(path):
343 parts = list(map(re.escape, path.split('*')))
344 for i in range(1, len(parts)-1):
345 parts[i] = f'(?>.*?{parts[i]})'
346 parts[-1] = f'.*{parts[-1]}'
347 return ''.join(parts)
348
349 def merge_entries(e1, e2):
350 entry = Entry()
351 entry.useragents = list(filter(set(e2.useragents).__contains__, e1.useragents))
352 entry.rulelines = e1.rulelines + e2.rulelines
353 entry.delay = e1.delay if e2.delay is None else e2.delay
354 entry.req_rate = e1.req_rate if e2.req_rate is None else e2.req_rate
355 return entry