| 1 |
# -*- coding: utf-8 -*- |
| 2 |
|
| 3 |
import pytest |
| 4 |
import requests |
| 5 |
import cloudscraper |
| 6 |
import cloudscraper.help as helper |
| 7 |
|
| 8 |
from collections import OrderedDict |
| 9 |
|
| 10 |
from cloudscraper.exceptions import ( |
| 11 |
CloudflareLoopProtection, |
| 12 |
CloudflareIUAMError, |
| 13 |
CloudflareCaptchaError, |
| 14 |
CloudflareCaptchaProvider, |
| 15 |
CaptchaParameter |
| 16 |
) |
| 17 |
|
| 18 |
from . import url, mockCloudflare |
| 19 |
|
| 20 |
|
| 21 |
class TestCloudScraper: |
| 22 |
|
| 23 |
# ------------------------------------------------------------------------------- # |
| 24 |
|
| 25 |
@mockCloudflare( |
| 26 |
fixture='js_challenge-27-05-2020.html', |
| 27 |
payload=OrderedDict([ |
| 28 |
('jschl_answer', '102.7365930923'), |
| 29 |
('pass', '1590526864.279-GCnY0hKdNH'), |
| 30 |
('jschl_vc', '706241acd6cb287b270835d26b275f50'), |
| 31 |
('r', 'b7f071b2f22b5dfecd144198317929d3f7c8d3d271c169e7676a42042c3581ed') |
| 32 |
]) |
| 33 |
) |
| 34 |
def test_js_challenge_27_05_2020(self, **kwargs): |
| 35 |
# test interpreters |
| 36 |
for interpreter in ['native', 'nodejs', 'js2py']: |
| 37 |
print('Testing {}'.format(interpreter)) |
| 38 |
scraper = cloudscraper.create_scraper(interpreter=interpreter, **kwargs) |
| 39 |
scraper.get(url) |
| 40 |
|
| 41 |
# ------------------------------------------------------------------------------- # |
| 42 |
|
| 43 |
@mockCloudflare( |
| 44 |
fixture='js_challenge1_16_05_2020.html', |
| 45 |
payload=OrderedDict([ |
| 46 |
('jschl_answer', '-7.5155218172'), |
| 47 |
('pass', '1589555973.262-n4Tt3w2mXt'), |
| 48 |
('jschl_vc', 'df9d3a159bbde53c214e7abfcb005e0c'), |
| 49 |
('r', 'f111bf7fe2d4d39d47dd837590808eeacb6e208a7503587f14a95c9ab0fc2d70') |
| 50 |
]) |
| 51 |
) |
| 52 |
def test_js_challenge1_16_05_2020(self, **kwargs): |
| 53 |
# test interpreters |
| 54 |
for interpreter in ['native', 'js2py', 'nodejs']: |
| 55 |
scraper = cloudscraper.create_scraper(interpreter=interpreter, **kwargs) |
| 56 |
scraper.get(url) |
| 57 |
|
| 58 |
# ------------------------------------------------------------------------------- # |
| 59 |
|
| 60 |
@mockCloudflare( |
| 61 |
fixture='js_challenge2_16_05_2020.html', |
| 62 |
payload=OrderedDict([ |
| 63 |
('jschl_answer', '58.4019148791'), |
| 64 |
('pass', '1589581670.18-mkcsKJpz0a'), |
| 65 |
('jschl_vc', 'ebec58fc1df89e2e403fa249469e50eb'), |
| 66 |
('r', 'af028b8967c508037f3888dfaf85d403df9532f4ad851fea76c2ece4f4d9853b') |
| 67 |
]) |
| 68 |
) |
| 69 |
def test_js_challenge2_16_05_2020(self, **kwargs): |
| 70 |
# test interpreters |
| 71 |
for interpreter in ['native', 'js2py', 'nodejs']: |
| 72 |
scraper = cloudscraper.create_scraper(interpreter=interpreter, **kwargs) |
| 73 |
scraper.get(url) |
| 74 |
|
| 75 |
# ------------------------------------------------------------------------------- # |
| 76 |
|
| 77 |
def test_create_scraper_js_challenge1_16_05_2020(self, **kwargs): |
| 78 |
# test |
| 79 |
session = requests.session() |
| 80 |
session.auth = ('user', 'test') |
| 81 |
cloudscraper.create_scraper(session) |
| 82 |
|
| 83 |
# ------------------------------------------------------------------------------- # |
| 84 |
|
| 85 |
@mockCloudflare(fixture='js_challenge1_16_05_2020.html', payload={}) |
| 86 |
def test_bad_interpreter_js_challenge1_16_05_2020(self, **kwargs): |
| 87 |
# test bad interpreter |
| 88 |
with pytest.raises( |
| 89 |
CloudflareIUAMError, |
| 90 |
match=r"Unable to parse Cloudflare anti-bots page: No module named*?" |
| 91 |
): |
| 92 |
scraper = cloudscraper.create_scraper(interpreter='badInterpreter', **kwargs) |
| 93 |
scraper.get(url) |
| 94 |
|
| 95 |
# ------------------------------------------------------------------------------- # |
| 96 |
|
| 97 |
@mockCloudflare( |
| 98 |
fixture='js_challenge1_16_05_2020.html', |
| 99 |
payload=OrderedDict([ |
| 100 |
('jschl_answer', 'NOPE'), |
| 101 |
('pass', '1589555973.262-n4Tt3w2mXt'), |
| 102 |
('jschl_vc', 'df9d3a159bbde53c214e7abfcb005e0c'), |
| 103 |
('r', 'f111bf7fe2d4d39d47dd837590808eeacb6e208a7503587f14a95c9ab0fc2d70') |
| 104 |
]) |
| 105 |
) |
| 106 |
def test_bad_solve_js_challenge1_16_05_2020(self, **kwargs): |
| 107 |
# test bad solve loop protection. |
| 108 |
with pytest.raises( |
| 109 |
CloudflareLoopProtection, |
| 110 |
match=r".*?Loop Protection.*?" |
| 111 |
): |
| 112 |
scraper = cloudscraper.create_scraper(**kwargs) |
| 113 |
scraper.get(url) |
| 114 |
|
| 115 |
# ------------------------------------------------------------------------------- # |
| 116 |
|
| 117 |
def test_bad_js_challenge1_16_05_2020(self, **kwargs): |
| 118 |
# test bad Captcha extraction. |
| 119 |
with pytest.raises( |
| 120 |
CloudflareIUAMError, |
| 121 |
match=r".*?we can't extract the parameters correctly.*?" |
| 122 |
): |
| 123 |
scraper = cloudscraper.create_scraper(**kwargs) |
| 124 |
cloudscraper.Cloudflare(scraper).IUAM_Challenge_Response('', '', '') |
| 125 |
|
| 126 |
with pytest.raises( |
| 127 |
CloudflareIUAMError, |
| 128 |
match=r"Cloudflare IUAM detected, unfortunately we can't extract the parameters correctly." |
| 129 |
): |
| 130 |
scraper = cloudscraper.create_scraper(**kwargs) |
| 131 |
cloudscraper.Cloudflare(scraper).IUAM_Challenge_Response( |
| 132 |
'id="challenge-form" action="blaaaaah" r value="" jschl_vc value="" pass value=""', |
| 133 |
url, |
| 134 |
'native' |
| 135 |
) |
| 136 |
|
| 137 |
# ------------------------------------------------------------------------------- # |
| 138 |
|
| 139 |
@mockCloudflare(fixture='reCaptcha_challenge_12_12_2019.html', payload={}) |
| 140 |
def test_Captcha_challenge_12_12_2019(self, **kwargs): |
| 141 |
# test bad reCaptcha detection. |
| 142 |
with pytest.raises( |
| 143 |
CloudflareCaptchaProvider, |
| 144 |
match=r".*?Captcha detected*?" |
| 145 |
): |
| 146 |
scraper = cloudscraper.create_scraper(**kwargs) |
| 147 |
scraper.get(url) |
| 148 |
|
| 149 |
scraper = cloudscraper.create_scraper(captcha={'provider': 'return_response'}, **kwargs) |
| 150 |
scraper.get(url) |
| 151 |
|
| 152 |
# ------------------------------------------------------------------------------- # |
| 153 |
|
| 154 |
def test_bad_reCaptcha_challenge_12_12_2019(self, **kwargs): |
| 155 |
# test bad reCaptcha extraction. |
| 156 |
with pytest.raises( |
| 157 |
CloudflareCaptchaError, |
| 158 |
match=r".*?we can't extract the parameters correctly.*?" |
| 159 |
): |
| 160 |
scraper = cloudscraper.create_scraper(**kwargs) |
| 161 |
cloudscraper.Cloudflare(scraper).captcha_Challenge_Response(None, None, '', '') |
| 162 |
|
| 163 |
# ------------------------------------------------------------------------------- # |
| 164 |
|
| 165 |
def test_user_agent(self, **kwargs): |
| 166 |
for browser in ['chrome', 'firefox']: |
| 167 |
scraper = cloudscraper.create_scraper(browser={'browser': browser, 'platform': 'windows'}, delay=0.1) |
| 168 |
assert browser in scraper.headers['User-Agent'].lower() |
| 169 |
|
| 170 |
# Check it can't find browsers.json |
| 171 |
with pytest.raises(RuntimeError, match=r"Sorry \"bad_match\" browser is not valid.*?"): |
| 172 |
scraper = cloudscraper.create_scraper(browser='bad_match', delay=0.1) |
| 173 |
|
| 174 |
# Check mobile and desktop disabled |
| 175 |
with pytest.raises( |
| 176 |
RuntimeError, |
| 177 |
match=r"Sorry you can't have mobile and desktop disabled at the same time\." |
| 178 |
): |
| 179 |
scraper = cloudscraper.create_scraper(browser={'browser': 'chrome', 'desktop': False, 'mobile': False}, delay=0.1) |
| 180 |
|
| 181 |
# check brotli |
| 182 |
scraper = cloudscraper.create_scraper(browser='chrome', allow_brotli=False, delay=0.1) |
| 183 |
assert 'br' not in scraper.headers['Accept-Encoding'] |
| 184 |
|
| 185 |
# test custom User-Agent |
| 186 |
scraper = cloudscraper.create_scraper(browser={'custom': 'test'}, delay=0.1) |
| 187 |
assert scraper.headers['User-Agent'] == 'test' |
| 188 |
|
| 189 |
# check its matched chrome and loaded correct cipherSuite |
| 190 |
scraper = cloudscraper.create_scraper(browser={'custom': '50.0.9370.394', 'tryMatchCustom': True}, delay=0.1) |
| 191 |
assert any('!' not in _ for _ in scraper.user_agent.cipherSuite) |
| 192 |
|
| 193 |
# check it didn't match anything and loaded custom cipherSuite |
| 194 |
scraper = cloudscraper.create_scraper(browser={'custom': 'aa50.0.9370.394', 'tryMatchCustom': True}, delay=0.1) |
| 195 |
assert any('!' in _ for _ in scraper.user_agent.cipherSuite) |
| 196 |
|
| 197 |
# ------------------------------------------------------------------------------- # |
| 198 |
# test ReCaptcha |
| 199 |
|
| 200 |
@mockCloudflare(fixture='reCaptcha_challenge_12_12_2019.html', payload={}) |
| 201 |
def test_reCaptcha_providers(self, **kwargs): |
| 202 |
for provider in ['9kw', '2captcha', 'anticaptcha', 'deathbycaptcha']: |
| 203 |
with pytest.raises( |
| 204 |
(CaptchaParameter, ImportError, CloudflareCaptchaError), |
| 205 |
match=r".*?: Missing .*? parameter\.|Please install.*?|Cloudflare Captcha detected.*?" |
| 206 |
): |
| 207 |
scraper = cloudscraper.create_scraper( |
| 208 |
captcha={ |
| 209 |
'provider': provider |
| 210 |
}, |
| 211 |
delay=0.1 |
| 212 |
) |
| 213 |
scraper.get(url) |
| 214 |
|
| 215 |
# ------------------------------------------------------------------------------- # |
| 216 |
|
| 217 |
def test_helper(self, **kwargs): |
| 218 |
payload = helper.systemInfo() |
| 219 |
assert payload |