Debian Codegraph @

tests/test_cloudscraper.py

raw
1 # -*- coding: utf-8 -*-
2
3 import pytest
4 import requests
5 import cloudscraper
6 import cloudscraper.help as helper
7
8 from collections import OrderedDict
9
10 from cloudscraper.exceptions import (
11 CloudflareLoopProtection,
12 CloudflareIUAMError,
13 CloudflareCaptchaError,
14 CloudflareCaptchaProvider,
15 CaptchaParameter
16 )
17
18 from . import url, mockCloudflare
19
20
21 class TestCloudScraper:
22
23 # ------------------------------------------------------------------------------- #
24
25 @mockCloudflare(
26 fixture='js_challenge-27-05-2020.html',
27 payload=OrderedDict([
28 ('jschl_answer', '102.7365930923'),
29 ('pass', '1590526864.279-GCnY0hKdNH'),
30 ('jschl_vc', '706241acd6cb287b270835d26b275f50'),
31 ('r', 'b7f071b2f22b5dfecd144198317929d3f7c8d3d271c169e7676a42042c3581ed')
32 ])
33 )
34 def test_js_challenge_27_05_2020(self, **kwargs):
35 # test interpreters
36 for interpreter in ['native', 'nodejs', 'js2py']:
37 print('Testing {}'.format(interpreter))
38 scraper = cloudscraper.create_scraper(interpreter=interpreter, **kwargs)
39 scraper.get(url)
40
41 # ------------------------------------------------------------------------------- #
42
43 @mockCloudflare(
44 fixture='js_challenge1_16_05_2020.html',
45 payload=OrderedDict([
46 ('jschl_answer', '-7.5155218172'),
47 ('pass', '1589555973.262-n4Tt3w2mXt'),
48 ('jschl_vc', 'df9d3a159bbde53c214e7abfcb005e0c'),
49 ('r', 'f111bf7fe2d4d39d47dd837590808eeacb6e208a7503587f14a95c9ab0fc2d70')
50 ])
51 )
52 def test_js_challenge1_16_05_2020(self, **kwargs):
53 # test interpreters
54 for interpreter in ['native', 'js2py', 'nodejs']:
55 scraper = cloudscraper.create_scraper(interpreter=interpreter, **kwargs)
56 scraper.get(url)
57
58 # ------------------------------------------------------------------------------- #
59
60 @mockCloudflare(
61 fixture='js_challenge2_16_05_2020.html',
62 payload=OrderedDict([
63 ('jschl_answer', '58.4019148791'),
64 ('pass', '1589581670.18-mkcsKJpz0a'),
65 ('jschl_vc', 'ebec58fc1df89e2e403fa249469e50eb'),
66 ('r', 'af028b8967c508037f3888dfaf85d403df9532f4ad851fea76c2ece4f4d9853b')
67 ])
68 )
69 def test_js_challenge2_16_05_2020(self, **kwargs):
70 # test interpreters
71 for interpreter in ['native', 'js2py', 'nodejs']:
72 scraper = cloudscraper.create_scraper(interpreter=interpreter, **kwargs)
73 scraper.get(url)
74
75 # ------------------------------------------------------------------------------- #
76
77 def test_create_scraper_js_challenge1_16_05_2020(self, **kwargs):
78 # test
79 session = requests.session()
80 session.auth = ('user', 'test')
81 cloudscraper.create_scraper(session)
82
83 # ------------------------------------------------------------------------------- #
84
85 @mockCloudflare(fixture='js_challenge1_16_05_2020.html', payload={})
86 def test_bad_interpreter_js_challenge1_16_05_2020(self, **kwargs):
87 # test bad interpreter
88 with pytest.raises(
89 CloudflareIUAMError,
90 match=r"Unable to parse Cloudflare anti-bots page: No module named*?"
91 ):
92 scraper = cloudscraper.create_scraper(interpreter='badInterpreter', **kwargs)
93 scraper.get(url)
94
95 # ------------------------------------------------------------------------------- #
96
97 @mockCloudflare(
98 fixture='js_challenge1_16_05_2020.html',
99 payload=OrderedDict([
100 ('jschl_answer', 'NOPE'),
101 ('pass', '1589555973.262-n4Tt3w2mXt'),
102 ('jschl_vc', 'df9d3a159bbde53c214e7abfcb005e0c'),
103 ('r', 'f111bf7fe2d4d39d47dd837590808eeacb6e208a7503587f14a95c9ab0fc2d70')
104 ])
105 )
106 def test_bad_solve_js_challenge1_16_05_2020(self, **kwargs):
107 # test bad solve loop protection.
108 with pytest.raises(
109 CloudflareLoopProtection,
110 match=r".*?Loop Protection.*?"
111 ):
112 scraper = cloudscraper.create_scraper(**kwargs)
113 scraper.get(url)
114
115 # ------------------------------------------------------------------------------- #
116
117 def test_bad_js_challenge1_16_05_2020(self, **kwargs):
118 # test bad Captcha extraction.
119 with pytest.raises(
120 CloudflareIUAMError,
121 match=r".*?we can't extract the parameters correctly.*?"
122 ):
123 scraper = cloudscraper.create_scraper(**kwargs)
124 cloudscraper.Cloudflare(scraper).IUAM_Challenge_Response('', '', '')
125
126 with pytest.raises(
127 CloudflareIUAMError,
128 match=r"Cloudflare IUAM detected, unfortunately we can't extract the parameters correctly."
129 ):
130 scraper = cloudscraper.create_scraper(**kwargs)
131 cloudscraper.Cloudflare(scraper).IUAM_Challenge_Response(
132 'id="challenge-form" action="blaaaaah" r value="" jschl_vc value="" pass value=""',
133 url,
134 'native'
135 )
136
137 # ------------------------------------------------------------------------------- #
138
139 @mockCloudflare(fixture='reCaptcha_challenge_12_12_2019.html', payload={})
140 def test_Captcha_challenge_12_12_2019(self, **kwargs):
141 # test bad reCaptcha detection.
142 with pytest.raises(
143 CloudflareCaptchaProvider,
144 match=r".*?Captcha detected*?"
145 ):
146 scraper = cloudscraper.create_scraper(**kwargs)
147 scraper.get(url)
148
149 scraper = cloudscraper.create_scraper(captcha={'provider': 'return_response'}, **kwargs)
150 scraper.get(url)
151
152 # ------------------------------------------------------------------------------- #
153
154 def test_bad_reCaptcha_challenge_12_12_2019(self, **kwargs):
155 # test bad reCaptcha extraction.
156 with pytest.raises(
157 CloudflareCaptchaError,
158 match=r".*?we can't extract the parameters correctly.*?"
159 ):
160 scraper = cloudscraper.create_scraper(**kwargs)
161 cloudscraper.Cloudflare(scraper).captcha_Challenge_Response(None, None, '', '')
162
163 # ------------------------------------------------------------------------------- #
164
165 def test_user_agent(self, **kwargs):
166 for browser in ['chrome', 'firefox']:
167 scraper = cloudscraper.create_scraper(browser={'browser': browser, 'platform': 'windows'}, delay=0.1)
168 assert browser in scraper.headers['User-Agent'].lower()
169
170 # Check it can't find browsers.json
171 with pytest.raises(RuntimeError, match=r"Sorry \"bad_match\" browser is not valid.*?"):
172 scraper = cloudscraper.create_scraper(browser='bad_match', delay=0.1)
173
174 # Check mobile and desktop disabled
175 with pytest.raises(
176 RuntimeError,
177 match=r"Sorry you can't have mobile and desktop disabled at the same time\."
178 ):
179 scraper = cloudscraper.create_scraper(browser={'browser': 'chrome', 'desktop': False, 'mobile': False}, delay=0.1)
180
181 # check brotli
182 scraper = cloudscraper.create_scraper(browser='chrome', allow_brotli=False, delay=0.1)
183 assert 'br' not in scraper.headers['Accept-Encoding']
184
185 # test custom User-Agent
186 scraper = cloudscraper.create_scraper(browser={'custom': 'test'}, delay=0.1)
187 assert scraper.headers['User-Agent'] == 'test'
188
189 # check its matched chrome and loaded correct cipherSuite
190 scraper = cloudscraper.create_scraper(browser={'custom': '50.0.9370.394', 'tryMatchCustom': True}, delay=0.1)
191 assert any('!' not in _ for _ in scraper.user_agent.cipherSuite)
192
193 # check it didn't match anything and loaded custom cipherSuite
194 scraper = cloudscraper.create_scraper(browser={'custom': 'aa50.0.9370.394', 'tryMatchCustom': True}, delay=0.1)
195 assert any('!' in _ for _ in scraper.user_agent.cipherSuite)
196
197 # ------------------------------------------------------------------------------- #
198 # test ReCaptcha
199
200 @mockCloudflare(fixture='reCaptcha_challenge_12_12_2019.html', payload={})
201 def test_reCaptcha_providers(self, **kwargs):
202 for provider in ['9kw', '2captcha', 'anticaptcha', 'deathbycaptcha']:
203 with pytest.raises(
204 (CaptchaParameter, ImportError, CloudflareCaptchaError),
205 match=r".*?: Missing .*? parameter\.|Please install.*?|Cloudflare Captcha detected.*?"
206 ):
207 scraper = cloudscraper.create_scraper(
208 captcha={
209 'provider': provider
210 },
211 delay=0.1
212 )
213 scraper.get(url)
214
215 # ------------------------------------------------------------------------------- #
216
217 def test_helper(self, **kwargs):
218 payload = helper.systemInfo()
219 assert payload