Debian Codegraph @

Lib/test/test_robotparser.py

raw
1 import io
2 import os
3 import threading
4 import unittest
5 import urllib.robotparser
6 from test import support
7 from test.support import socket_helper
8 from test.support import threading_helper
9 from http.server import BaseHTTPRequestHandler, HTTPServer
10
11
12 class BaseRobotTest:
13 robots_txt = ''
14 agent = 'test_robotparser'
15 good = []
16 bad = []
17 site_maps = None
18 expected_output = None
19
20 def __init_subclass__(cls):
21 super().__init_subclass__()
22 # Remove tests that do nothing.
23 if issubclass(cls, unittest.TestCase):
24 if not cls.good:
25 cls.test_good_urls = None
26 if not cls.bad:
27 cls.test_bad_urls = None
28 if cls.expected_output is None:
29 cls.test_string_formatting = None
30
31 def setUp(self):
32 lines = io.StringIO(self.robots_txt).readlines()
33 self.parser = urllib.robotparser.RobotFileParser()
34 self.parser.parse(lines)
35
36 def get_agent_and_url(self, url):
37 if isinstance(url, tuple):
38 agent, url = url
39 return agent, url
40 return self.agent, url
41
42 def test_good_urls(self):
43 for url in self.good:
44 agent, url = self.get_agent_and_url(url)
45 with self.subTest(url=url, agent=agent):
46 self.assertTrue(self.parser.can_fetch(agent, url))
47
48 def test_bad_urls(self):
49 for url in self.bad:
50 agent, url = self.get_agent_and_url(url)
51 with self.subTest(url=url, agent=agent):
52 self.assertFalse(self.parser.can_fetch(agent, url))
53
54 def test_site_maps(self):
55 self.assertEqual(self.parser.site_maps(), self.site_maps)
56
57 def test_string_formatting(self):
58 self.assertEqual(str(self.parser), self.expected_output)
59
60 class UserAgentWildcardTest(BaseRobotTest, unittest.TestCase):
61 robots_txt = """\
62 User-agent: *
63 Disallow: /cyberworld/map/ # This is an infinite virtual URL space
64 Disallow: /tmp/ # these will soon disappear
65 Disallow: /foo.html
66 """
67 good = ['/', '/test.html']
68 bad = ['/cyberworld/map/index.html', '/tmp/xxx', '/foo.html']
69
70 class SimpleExampleTest(BaseRobotTest, unittest.TestCase):
71 # Example from RFC 9309, section 5.1.
72 robots_txt = """\
73 User-Agent: *
74 Disallow: *.gif$
75 Disallow: /example/
76 Allow: /publications/
77
78 User-Agent: foobot
79 Disallow:/
80 Allow:/example/page.html
81 Allow:/example/allowed.gif
82
83 User-Agent: barbot
84 User-Agent: bazbot
85 Disallow: /example/page.html
86
87 User-Agent: quxbot
88 """
89 good = [
90 '/', '/publications/',
91 ('foobot', '/example/page.html'), ('foobot', '/example/allowed.gif'),
92 ('barbot', '/'), ('barbot', '/example/'),
93 ('barbot', '/example/allowed.gif'),
94 ('barbot', '/example/disallowed.gif'),
95 ('barbot', '/publications/'),
96 ('barbot', '/publications/allowed.gif'),
97 ('bazbot', '/'), ('bazbot', '/example/'),
98 ('bazbot', '/example/allowed.gif'),
99 ('bazbot', '/example/disallowed.gif'),
100 ('bazbot', '/publications/'),
101 ('bazbot', '/publications/allowed.gif'),
102 ('quxbot', '/'), ('quxbot', '/example/'),
103 ('quxbot', '/example/page.html'), ('quxbot', '/example/allowed.gif'),
104 ('quxbot', '/example/disallowed.gif'),
105 ('quxbot', '/publications/'),
106 ('quxbot', '/publications/allowed.gif'),
107 ]
108 bad = [
109 '/example/', '/example/page.html', '/example/allowed.gif',
110 '/example/disallowed.gif',
111 '/publications/allowed.gif',
112 ('foobot', '/'), ('foobot', '/example/'),
113 ('foobot', '/example/disallowed.gif'),
114 ('foobot', '/publications/'),
115 ('foobot', '/publications/allowed.gif'),
116 ('barbot', '/example/page.html'),
117 ('bazbot', '/example/page.html'),
118 ]
119
120
121 class CrawlDelayAndCustomAgentTest(BaseRobotTest, unittest.TestCase):
122 robots_txt = """\
123 # robots.txt for http://www.example.com/
124
125 User-agent: *
126 Crawl-delay: 1
127 Request-rate: 3/15
128 Disallow: /cyberworld/map/ # This is an infinite virtual URL space
129
130 # Cybermapper knows where to go.
131 User-agent: cybermapper
132 Disallow:
133 """
134 good = ['/', '/test.html', ('cybermapper', '/cyberworld/map/index.html')]
135 bad = ['/cyberworld/map/index.html']
136
137
138 class SitemapTest(BaseRobotTest, unittest.TestCase):
139 robots_txt = """\
140 # robots.txt for http://www.example.com/
141
142 User-agent: *
143 Sitemap: http://www.gstatic.com/s2/sitemaps/profiles-sitemap.xml
144 Sitemap: http://www.google.com/hostednews/sitemap_index.xml
145 Request-rate: 3/15
146 Disallow: /cyberworld/map/ # This is an infinite virtual URL space
147
148 """
149 good = ['/', '/test.html']
150 bad = ['/cyberworld/map/index.html']
151 site_maps = ['http://www.gstatic.com/s2/sitemaps/profiles-sitemap.xml',
152 'http://www.google.com/hostednews/sitemap_index.xml']
153
154
155 class RejectAllRobotsTest(BaseRobotTest, unittest.TestCase):
156 robots_txt = """\
157 # go away
158 User-agent: *
159 Disallow: /
160 """
161 good = ['/robots.txt']
162 bad = ['/cyberworld/map/index.html', '/', '/tmp/']
163
164
165 class BaseRequestRateTest(BaseRobotTest):
166 request_rate = None
167 crawl_delay = None
168
169 def test_request_rate(self):
170 parser = self.parser
171 for url in self.good + self.bad:
172 agent, url = self.get_agent_and_url(url)
173 with self.subTest(url=url, agent=agent):
174 self.assertEqual(parser.crawl_delay(agent), self.crawl_delay)
175
176 parsed_request_rate = parser.request_rate(agent)
177 self.assertEqual(parsed_request_rate, self.request_rate)
178 if self.request_rate is not None:
179 self.assertIsInstance(
180 parsed_request_rate,
181 urllib.robotparser.RequestRate
182 )
183 self.assertEqual(
184 parsed_request_rate.requests,
185 self.request_rate.requests
186 )
187 self.assertEqual(
188 parsed_request_rate.seconds,
189 self.request_rate.seconds
190 )
191
192
193 class EmptyFileTest(BaseRequestRateTest, unittest.TestCase):
194 robots_txt = ''
195 good = ['/foo']
196 expected_output = ''
197
198
199 class CrawlDelayAndRequestRateTest(BaseRequestRateTest, unittest.TestCase):
200 robots_txt = """\
201 User-agent: figtree
202 Crawl-delay: 3
203 Request-rate: 9/30
204 Disallow: /tmp
205 Disallow: /a%3cd.html
206 Disallow: /a%2fb.html
207 Disallow: /%7ejoe/index.html
208 """
209 agent = 'figtree'
210 request_rate = urllib.robotparser.RequestRate(9, 30)
211 crawl_delay = 3
212 good = [('figtree', '/foo.html')]
213 bad = ['/tmp', '/tmp.html', '/tmp/a.html', '/a%3cd.html', '/a%3Cd.html',
214 '/a%2fb.html', '/~joe/index.html']
215
216
217 class DifferentAgentTest(CrawlDelayAndRequestRateTest):
218 agent = 'FigTree Robot libwww-perl/5.04'
219
220
221 class InvalidRequestRateTest(BaseRobotTest, unittest.TestCase):
222 robots_txt = """\
223 User-agent: *
224 Disallow: /tmp/
225 Disallow: /a%3Cd.html
226 Disallow: /a/b.html
227 Disallow: /%7ejoe/index.html
228 Crawl-delay: 3
229 Request-rate: 9/banana
230 """
231 good = ['/tmp']
232 bad = ['/tmp/', '/tmp/a.html', '/a%3cd.html', '/a%3Cd.html', '/a/b.html',
233 '/%7Ejoe/index.html']
234 crawl_delay = 3
235
236
237 class InvalidCrawlDelayTest(BaseRobotTest, unittest.TestCase):
238 # From bug report #523041
239 robots_txt = """\
240 User-Agent: *
241 Disallow: /.
242 Crawl-delay: pears
243 """
244 good = ['/foo.html']
245 # bug report says "/" should be denied, but that is not in the RFC
246 bad = []
247
248
249 class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase):
250 # also test that Allow and Diasallow works well with each other
251 robots_txt = """\
252 User-agent: Googlebot
253 Allow: /folder1/myfile.html
254 Disallow: /folder1/
255 Request-rate: whale/banana
256 """
257 agent = 'Googlebot'
258 good = ['/folder1/myfile.html']
259 bad = ['/folder1/anotherfile.html']
260
261
262 class UserAgentOrderingTest(BaseRobotTest, unittest.TestCase):
263 # the order of User-agent should not matter
264 robots_txt = """\
265 User-agent: Googlebot
266 Disallow: /
267 Allow: /folder1/
268
269 User-agent: Googlebot-Mobile
270 Allow: /
271 Disallow: /folder1/
272 """
273 agent = 'Googlebot'
274 bad = ['/something.jpg']
275 good = ['/folder1/myfile.html']
276
277
278 class UserAgentGoogleMobileTest(UserAgentOrderingTest):
279 agent = 'Googlebot-mobile'
280 bad = ['/folder1/myfile.html']
281 good = ['/something.jpg']
282
283
284 class LongestMatchTest(BaseRobotTest, unittest.TestCase):
285 # Based on example from RFC 9309, section 5.2.
286 robots_txt = """\
287 User-agent: *
288 Allow: /example/page/
289 Disallow: /example/page/disallowed.gif
290 Allow: /example/
291 """
292 good = ['/example/', '/example/page/']
293 bad = ['/example/page/disallowed.gif']
294
295
296 class LongestMatchWildcardTest(BaseRobotTest, unittest.TestCase):
297 robots_txt = """\
298 User-agent: *
299 Allow: /example/page/
300 Disallow: *.gif
301 Allow: /example/
302 """
303 good = ['/example/', '/example/page/']
304 bad = ['/example/page/disallowed.gif', '/x.gif']
305
306
307 class AllowWinsEqualMatchTest(BaseRobotTest, unittest.TestCase):
308 robots_txt = """\
309 User-agent: *
310 Disallow: /spam
311 Allow: /spam
312 Disallow: /spam
313 """
314 good = ['/spam', '/spam/']
315
316
317 class AllowWinsEqualFullMatchTest(BaseRobotTest, unittest.TestCase):
318 robots_txt = """\
319 User-agent: *
320 Disallow: /spam
321 Allow: /spam$
322 Disallow: /spam
323 Disallow: /eggs$
324 Allow: /eggs
325 Disallow: /eggs$
326 """
327 good = ['/spam', '/eggs', '/eggs/']
328 bad = ['/spam/']
329
330
331 class AllowWinsEqualMatchWildcardTest(BaseRobotTest, unittest.TestCase):
332 robots_txt = """\
333 User-agent: *
334 Disallow: /spam
335 Allow: *am
336 Disallow: /spam
337 Disallow: *gs
338 Allow: /eggs
339 Disallow: *gs
340 """
341 good = ['/spam', '/eggs', '/spam/', '/eggs/']
342
343
344 class MergeGroupsTest(BaseRobotTest, unittest.TestCase):
345 robots_txt = """\
346 User-agent: spambot
347 Disallow: /some/path
348
349 User-agent: spambot
350 Disallow: /another/path
351 """
352 agent = 'spambot'
353 bad = ['/some/path', '/another/path']
354
355
356 class UserAgentStartsGroupTest(BaseRobotTest, unittest.TestCase):
357 robots_txt = """\
358 User-agent: spambot
359 Disallow: /some/path
360 User-agent: eggsbot
361 Disallow: /another/path
362 """
363 good = [('spambot', '/'), ('spambot', '/another/path'),
364 ('eggsbot', '/'), ('eggsbot', '/some/path')]
365 bad = [('spambot', '/some/path'), ('eggsbot', '/another/path')]
366 expected_output = """\
367 User-agent: spambot
368 Disallow: /some/path
369
370 User-agent: eggsbot
371 Disallow: /another/path\
372 """
373
374 class IgnoreEmptyLinesTest(BaseRobotTest, unittest.TestCase):
375 robots_txt = """\
376 User-agent: spambot
377
378 User-agent: eggsbot
379 Disallow: /some/path
380
381 Disallow: /another/path
382 """
383 good = [('spambot', '/'), ('eggsbot', '/')]
384 bad = [
385 ('spambot', '/some/path'), ('spambot', '/another/path'),
386 ('eggsbot', '/some/path'), ('eggsbot', '/another/path'),
387 ]
388 expected_output = """\
389 User-agent: spambot
390 User-agent: eggsbot
391 Disallow: /some/path
392 Disallow: /another/path\
393 """
394
395
396 class IgnoreRulesWithoutUserAgentTest(BaseRobotTest, unittest.TestCase):
397 robots_txt = """\
398 Disallow: /some/path
399
400 User-agent: *
401 Disallow: /another/path
402 """
403 good = ['/', '/some/path']
404 bad = ['/another/path']
405 expected_output = """\
406 User-agent: *
407 Disallow: /another/path\
408 """
409
410
411 class EmptyGroupTest(BaseRobotTest, unittest.TestCase):
412 robots_txt = """\
413 User-agent: *
414 Disallow: /some/path
415
416 User-agent: spambot
417 """
418 agent = 'spambot'
419 good = ['/', '/some/path']
420 expected_output = """\
421 User-agent: *
422 Disallow: /some/path
423
424 User-agent: spambot
425 Allow:\
426 """
427
428
429 class WeirdPathTest(BaseRobotTest, unittest.TestCase):
430 robots_txt = f"""\
431 User-agent: *
432 Disallow: /a$$$
433 Disallow: /b$z
434 Disallow: /c***
435 Disallow: /d***z
436 Disallow: /e*$**$$
437 Disallow: /f*$**$$z
438 Disallow: /g$*$$**
439 Disallow: /h$*$$**z
440 """
441 good = ['/ax', '/a$$', '/b', '/bz', '/b$z', '/d', '/f', '/fz',
442 '/f$$$z', '/fx$y$$z', '/gx', '/g$$$', '/g$x$$y', '/h', '/hz',
443 '/h$$$z', '/h$x$$yz']
444 bad = ['/a', '/c', '/cxy', '/dz', '/dxyz', '/dxzy', '/e', '/exy',
445 '/e$$', '/ex$y$', '/g']
446 expected_output = """\
447 User-agent: *
448 Disallow: /a$
449 Disallow: /c*
450 Disallow: /d*z
451 Disallow: /e*$
452 Disallow: /g$\
453 """
454
455
456 class PathWithManyWildcardsTest(BaseRobotTest, unittest.TestCase):
457 # This test would take many years if use naive translation to regular
458 # expression (* -> .*).
459 N = 50
460 robots_txt = f"""\
461 User-agent: *
462 Disallow: /{'*a'*N}*b
463 """
464 good = ['/' + 'a'*N + 'a']
465 bad = ['/' + 'a'*N + 'b']
466
467
468 class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase):
469 # see issue #6325 for details
470 robots_txt = """\
471 User-agent: *
472 Disallow: /some/path?name=value
473 Disallow: /another/path?
474 Disallow: /yet/one/path?name=value&more
475 """
476 good = ['/some/path', '/some/path?',
477 '/some/path%3Fname=value', '/some/path?name%3Dvalue',
478 '/another/path', '/another/path%3F',
479 '/yet/one/path?name=value%26more',
480 '/some/pathxname=value']
481 bad = ['/some/path?name=value'
482 '/another/path?', '/another/path?name=value',
483 '/yet/one/path?name=value&more']
484
485
486 class PercentEncodingTest(BaseRobotTest, unittest.TestCase):
487 robots_txt = """\
488 User-agent: *
489 Disallow: /a1/Z-._~ # unreserved characters
490 Disallow: /a2/%5A%2D%2E%5F%7E # percent-encoded unreserved characters
491 Disallow: /u1/%F0%9F%90%8D # percent-encoded ASCII Unicode character
492 Disallow: /u2/%f0%9f%90%8d
493 Disallow: /u3/\U0001f40d # raw non-ASCII Unicode character
494 Disallow: /v1/%F0 # percent-encoded non-ASCII octet
495 Disallow: /v2/%f0
496 Disallow: /v3/\udcf0 # raw non-ASCII octet
497 Disallow: /p1%xy # raw percent
498 Disallow: /p2%
499 Disallow: /p3%25xy # percent-encoded percent
500 Disallow: /p4%2525xy # double percent-encoded percent
501 Disallow: /john%20smith # space
502 Disallow: /john doe
503 Disallow: /trailingspace%20
504 Disallow: /question%3Fq=v # not query
505 Disallow: /hash%23f # not fragment
506 Disallow: /dollar%24
507 Disallow: /asterisk%2A
508 Disallow: /sub/dir
509 Disallow: /slash%2F
510 Disallow: /query/question?q=%3F
511 Disallow: /query/raw/question?q=?
512 Disallow: /query/eq?q%3Dv
513 Disallow: /query/amp?q=v%26a
514 """
515 good = [
516 '/u1/%F0', '/u1/%f0',
517 '/u2/%F0', '/u2/%f0',
518 '/u3/%F0', '/u3/%f0',
519 '/p1%2525xy', '/p2%f0', '/p3%2525xy', '/p4%xy', '/p4%25xy',
520 '/question?q=v',
521 '/dollar', '/asterisk',
522 '/query/eq?q=v',
523 '/query/amp?q=v&a',
524 ]
525 bad = [
526 '/a1/Z-._~', '/a1/%5A%2D%2E%5F%7E',
527 '/a2/Z-._~', '/a2/%5A%2D%2E%5F%7E',
528 '/u1/%F0%9F%90%8D', '/u1/%f0%9f%90%8d', '/u1/\U0001f40d',
529 '/u2/%F0%9F%90%8D', '/u2/%f0%9f%90%8d', '/u2/\U0001f40d',
530 '/u3/%F0%9F%90%8D', '/u3/%f0%9f%90%8d', '/u3/\U0001f40d',
531 '/v1/%F0', '/v1/%f0', '/v1/\udcf0', '/v1/\U0001f40d',
532 '/v2/%F0', '/v2/%f0', '/v2/\udcf0', '/v2/\U0001f40d',
533 '/v3/%F0', '/v3/%f0', '/v3/\udcf0', '/v3/\U0001f40d',
534 '/p1%xy', '/p1%25xy',
535 '/p2%', '/p2%25', '/p2%2525', '/p2%xy',
536 '/p3%xy', '/p3%25xy',
537 '/p4%2525xy',
538 '/john%20smith', '/john smith',
539 '/john%20doe', '/john doe',
540 '/trailingspace%20', '/trailingspace ',
541 '/question%3Fq=v',
542 '/hash#f', '/hash%23f',
543 '/dollar$', '/dollar%24',
544 '/asterisk*', '/asterisk%2A',
545 '/sub/dir', '/sub%2Fdir',
546 '/slash%2F', '/slash/',
547 '/query/question?q=?', '/query/question?q=%3F',
548 '/query/raw/question?q=?', '/query/raw/question?q=%3F',
549 '/query/eq?q%3Dv',
550 '/query/amp?q=v%26a',
551 ]
552 # other reserved characters
553 for c in ":/#[]@!$&'()*+,;=":
554 robots_txt += f'Disallow: /raw{c}\nDisallow: /pc%{ord(c):02X}\n'
555 bad.append(f'/raw{c}')
556 bad.append(f'/raw%{ord(c):02X}')
557 bad.append(f'/pc{c}')
558 bad.append(f'/pc%{ord(c):02X}')
559
560
561 class DefaultEntryTest(BaseRequestRateTest, unittest.TestCase):
562 robots_txt = """\
563 User-agent: *
564 Crawl-delay: 1
565 Request-rate: 3/15
566 Disallow: /cyberworld/map/
567 """
568 request_rate = urllib.robotparser.RequestRate(3, 15)
569 crawl_delay = 1
570 good = ['/', '/test.html']
571 bad = ['/cyberworld/map/index.html']
572
573
574 class StringFormattingTest(BaseRobotTest, unittest.TestCase):
575 robots_txt = """\
576 User-agent: *
577 Crawl-delay: 1
578 Request-rate: 3/15
579 Disallow: /cyberworld/map/ # This is an infinite virtual URL space
580
581 # Cybermapper knows where to go.
582 User-agent: cybermapper
583 Disallow: /some/path
584 """
585
586 expected_output = """\
587 User-agent: *
588 Crawl-delay: 1
589 Request-rate: 3/15
590 Disallow: /cyberworld/map/
591
592 User-agent: cybermapper
593 Disallow: /some/path\
594 """
595
596
597 class ConstructedStringFormattingTest(unittest.TestCase):
598 def test_empty(self):
599 parser = urllib.robotparser.RobotFileParser()
600 self.assertEqual(str(parser), '')
601
602 def test_group_without_rules(self):
603 parser = urllib.robotparser.RobotFileParser()
604 entry = urllib.robotparser.Entry()
605 entry.useragents = ['spambot']
606 parser._add_entry(entry)
607 entry = urllib.robotparser.Entry()
608 entry.useragents = ['hambot']
609 entry.rulelines = [urllib.robotparser.RuleLine('/ham', False)]
610 parser._add_entry(entry)
611 entry = urllib.robotparser.Entry()
612 entry.useragents = ['eggsbot']
613 parser._add_entry(entry)
614 self.assertEqual(str(parser), """\
615 User-agent: spambot
616 Allow:
617
618 User-agent: hambot
619 Disallow: /ham
620
621 User-agent: eggsbot
622 Allow:\
623 """)
624
625 def test_group_without_user_agent(self):
626 parser = urllib.robotparser.RobotFileParser()
627 entry = urllib.robotparser.Entry()
628 entry.rulelines = [urllib.robotparser.RuleLine('/ham', False)]
629 parser._add_entry(entry)
630 entry = urllib.robotparser.Entry()
631 entry.useragents = ['spambot']
632 entry.rulelines = [urllib.robotparser.RuleLine('/spam', False)]
633 parser._add_entry(entry)
634 entry = urllib.robotparser.Entry()
635 entry.rulelines = [urllib.robotparser.RuleLine('/eggs', False)]
636 parser._add_entry(entry)
637 self.assertEqual(str(parser), """\
638 User-agent: spambot
639 Disallow: /spam\
640 """)
641
642
643 @unittest.skipUnless(
644 support.has_socket_support,
645 "Socket server requires working socket."
646 )
647 class BaseLocalNetworkTestCase:
648
649 @classmethod
650 def setUpClass(cls):
651 # clear _opener global variable
652 cls.addClassCleanup(urllib.request.urlcleanup)
653
654 cls.server = HTTPServer((socket_helper.HOST, 0), cls.RobotHandler)
655 cls.addClassCleanup(cls.server.server_close)
656
657 t = threading.Thread(
658 name='HTTPServer serving',
659 target=cls.server.serve_forever,
660 # Short poll interval to make the test finish quickly.
661 # Time between requests is short enough that we won't wake
662 # up spuriously too many times.
663 kwargs={'poll_interval':0.01})
664 cls.enterClassContext(threading_helper.start_threads([t]))
665 cls.addClassCleanup(cls.server.shutdown)
666
667
668 SAMPLE_ROBOTS_TXT = b'''\
669 User-agent: test_robotparser
670 Disallow: /utf8/\xf0\x9f\x90\x8d
671 Disallow: /non-utf8/\xf0
672 Disallow: //[spam]/path
673 '''
674
675
676 class LocalNetworkTestCase(BaseLocalNetworkTestCase, unittest.TestCase):
677 class RobotHandler(BaseHTTPRequestHandler):
678
679 def do_GET(self):
680 self.send_response(200)
681 self.end_headers()
682 self.wfile.write(SAMPLE_ROBOTS_TXT)
683
684 def log_message(self, format, *args):
685 pass
686
687 def testRead(self):
688 # Test that reading a weird robots.txt doesn't fail.
689 addr = self.server.server_address
690 url = f'http://{socket_helper.HOST}:{addr[1]}'
691 robots_url = url + '/robots.txt'
692 parser = urllib.robotparser.RobotFileParser()
693 parser.set_url(robots_url)
694 parser.read()
695 # And it can even interpret the weird paths in some reasonable way.
696 agent = 'test_robotparser'
697 self.assertTrue(parser.can_fetch(agent, robots_url))
698 self.assertTrue(parser.can_fetch(agent, url + '/utf8/'))
699 self.assertFalse(parser.can_fetch(agent, url + '/utf8/\U0001f40d'))
700 self.assertFalse(parser.can_fetch(agent, url + '/utf8/%F0%9F%90%8D'))
701 self.assertTrue(parser.can_fetch(agent, url + '/non-utf8/'))
702 self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/%F0'))
703 self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/\U0001f40d'))
704 self.assertFalse(parser.can_fetch(agent, url + '/%2F[spam]/path'))
705
706
707 class HttpErrorsTestCase(BaseLocalNetworkTestCase, unittest.TestCase):
708 class RobotHandler(BaseHTTPRequestHandler):
709
710 def do_GET(self):
711 self.send_error(self.server.return_code)
712
713 def log_message(self, format, *args):
714 pass
715
716 def setUp(self):
717 # Make sure that a valid code is set in the test.
718 self.server.return_code = None
719
720 def testUnauthorized(self):
721 self.server.return_code = 401
722 addr = self.server.server_address
723 url = f'http://{socket_helper.HOST}:{addr[1]}'
724 robots_url = url + "/robots.txt"
725 parser = urllib.robotparser.RobotFileParser()
726 parser.set_url(url)
727 parser.read()
728 self.assertFalse(parser.can_fetch("*", robots_url))
729 self.assertFalse(parser.can_fetch("*", url + '/some/file.html'))
730
731 def testForbidden(self):
732 self.server.return_code = 403
733 addr = self.server.server_address
734 url = f'http://{socket_helper.HOST}:{addr[1]}'
735 robots_url = url + "/robots.txt"
736 parser = urllib.robotparser.RobotFileParser()
737 parser.set_url(url)
738 parser.read()
739 self.assertFalse(parser.can_fetch("*", robots_url))
740 self.assertFalse(parser.can_fetch("*", url + '/some/file.html'))
741
742 def testNotFound(self):
743 self.server.return_code = 404
744 addr = self.server.server_address
745 url = f'http://{socket_helper.HOST}:{addr[1]}'
746 robots_url = url + "/robots.txt"
747 parser = urllib.robotparser.RobotFileParser()
748 parser.set_url(url)
749 parser.read()
750 self.assertTrue(parser.can_fetch("*", robots_url))
751 self.assertTrue(parser.can_fetch("*", url + '/path/file.html'))
752
753 def testTeapot(self):
754 self.server.return_code = 418
755 addr = self.server.server_address
756 url = f'http://{socket_helper.HOST}:{addr[1]}'
757 robots_url = url + "/robots.txt"
758 parser = urllib.robotparser.RobotFileParser()
759 parser.set_url(url)
760 parser.read()
761 self.assertTrue(parser.can_fetch("*", robots_url))
762 self.assertTrue(parser.can_fetch("*", url + '/pot-1?milk-type=Cream'))
763
764 def testServiceUnavailable(self):
765 self.server.return_code = 503
766 addr = self.server.server_address
767 url = f'http://{socket_helper.HOST}:{addr[1]}'
768 robots_url = url + "/robots.txt"
769 parser = urllib.robotparser.RobotFileParser()
770 parser.set_url(url)
771 parser.read()
772 self.assertFalse(parser.can_fetch("*", robots_url))
773 self.assertFalse(parser.can_fetch("*", url + '/path/file.html'))
774
775
776 @support.requires_working_socket()
777 class NetworkTestCase(unittest.TestCase):
778
779 base_url = 'http://www.pythontest.net/'
780 robots_txt = '{}elsewhere/robots.txt'.format(base_url)
781
782 @classmethod
783 def setUpClass(cls):
784 support.requires('network')
785 cls.addClassCleanup(urllib.request.urlcleanup)
786 with socket_helper.transient_internet(cls.base_url):
787 cls.parser = urllib.robotparser.RobotFileParser(cls.robots_txt)
788 cls.parser.read()
789
790 def url(self, path):
791 return '{}{}{}'.format(
792 self.base_url, path, '/' if not os.path.splitext(path)[1] else ''
793 )
794
795 def test_basic(self):
796 self.assertFalse(self.parser.disallow_all)
797 self.assertFalse(self.parser.allow_all)
798 self.assertGreater(self.parser.mtime(), 0)
799 self.assertFalse(self.parser.crawl_delay('*'))
800 self.assertFalse(self.parser.request_rate('*'))
801
802 def test_can_fetch(self):
803 self.assertTrue(self.parser.can_fetch('*', self.url('elsewhere')))
804 self.assertFalse(self.parser.can_fetch('Nutch', self.base_url))
805 self.assertTrue(self.parser.can_fetch('Nutch', self.url('brian')))
806 self.assertFalse(self.parser.can_fetch('Nutch', self.url('webstats')))
807 self.assertFalse(self.parser.can_fetch('*', self.url('webstats')))
808 self.assertTrue(self.parser.can_fetch('*', self.base_url))
809
810 def test_read_404(self):
811 parser = urllib.robotparser.RobotFileParser(self.url('i-robot.txt'))
812 parser.read()
813 self.assertTrue(parser.allow_all)
814 self.assertFalse(parser.disallow_all)
815 self.assertEqual(parser.mtime(), 0)
816 self.assertIsNone(parser.crawl_delay('*'))
817 self.assertIsNone(parser.request_rate('*'))
818
819 if __name__=='__main__':
820 unittest.main()