Debian Codegraph @

cloudscraper/cloudflare.py

raw
1 # Cloudflare V1
2
3 import re
4 import sys
5 import time
6
7 from copy import deepcopy
8 from collections import OrderedDict
9
10 # ------------------------------------------------------------------------------- #
11
12 try:
13 from HTMLParser import HTMLParser
14 except ImportError:
15 if sys.version_info >= (3, 4):
16 import html
17 else:
18 from html.parser import HTMLParser
19
20 try:
21 from urlparse import urlparse, urljoin
22 except ImportError:
23 from urllib.parse import urlparse, urljoin
24
25 # ------------------------------------------------------------------------------- #
26
27 from .exceptions import (
28 CloudflareCode1020,
29 CloudflareIUAMError,
30 CloudflareSolveError,
31 CloudflareChallengeError,
32 CloudflareCaptchaError,
33 CloudflareCaptchaProvider
34 )
35
36 # ------------------------------------------------------------------------------- #
37
38 from .captcha import Captcha
39 from .interpreters import JavaScriptInterpreter
40
41 # ------------------------------------------------------------------------------- #
42
43
44 class Cloudflare():
45
46 def __init__(self, cloudscraper):
47 self.cloudscraper = cloudscraper
48
49 # ------------------------------------------------------------------------------- #
50 # Unescape / decode html entities
51 # ------------------------------------------------------------------------------- #
52
53 @staticmethod
54 def unescape(html_text):
55 if sys.version_info >= (3, 0):
56 if sys.version_info >= (3, 4):
57 return html.unescape(html_text)
58
59 return HTMLParser().unescape(html_text)
60
61 return HTMLParser().unescape(html_text)
62
63 # ------------------------------------------------------------------------------- #
64 # check if the response contains a valid Cloudflare challenge
65 # ------------------------------------------------------------------------------- #
66
67 @staticmethod
68 def is_IUAM_Challenge(resp):
69 try:
70 return (
71 resp.headers.get('Server', '').startswith('cloudflare')
72 and resp.status_code in [429, 503]
73 and re.search(r'/cdn-cgi/images/trace/jsch/', resp.text, re.M | re.S)
74 and re.search(
75 r'''<form .*?="challenge-form" action="/\S+__cf_chl_f_tk=''',
76 resp.text,
77 re.M | re.S
78 )
79 )
80 except AttributeError:
81 pass
82
83 return False
84
85 # ------------------------------------------------------------------------------- #
86 # check if the response contains new Cloudflare challenge
87 # ------------------------------------------------------------------------------- #
88
89 def is_New_IUAM_Challenge(self, resp):
90 try:
91 return (
92 self.is_IUAM_Challenge(resp)
93 and re.search(
94 r'''cpo.src\s*=\s*['"]/cdn-cgi/challenge-platform/\S+orchestrate/jsch/v1''',
95 resp.text,
96 re.M | re.S
97 )
98 )
99 except AttributeError:
100 pass
101
102 return False
103
104 # ------------------------------------------------------------------------------- #
105 # check if the response contains a v2 hCaptcha Cloudflare challenge
106 # ------------------------------------------------------------------------------- #
107
108 def is_New_Captcha_Challenge(self, resp):
109 try:
110 return (
111 self.is_Captcha_Challenge(resp)
112 and re.search(
113 r'''cpo.src\s*=\s*['"]/cdn-cgi/challenge-platform/\S+orchestrate/(captcha|managed)/v1''',
114 resp.text,
115 re.M | re.S
116 )
117 )
118 except AttributeError:
119 pass
120
121 return False
122
123 # ------------------------------------------------------------------------------- #
124 # check if the response contains a Cloudflare hCaptcha challenge
125 # ------------------------------------------------------------------------------- #
126
127 @staticmethod
128 def is_Captcha_Challenge(resp):
129 try:
130 return (
131 resp.headers.get('Server', '').startswith('cloudflare')
132 and resp.status_code == 403
133 and re.search(r'/cdn-cgi/images/trace/(captcha|managed)/', resp.text, re.M | re.S)
134 and re.search(
135 r'''<form .*?="challenge-form" action="/\S+__cf_chl_f_tk=''',
136 resp.text,
137 re.M | re.S
138 )
139 )
140 except AttributeError:
141 pass
142
143 return False
144
145 # ------------------------------------------------------------------------------- #
146 # check if the response contains Firewall 1020 Error
147 # ------------------------------------------------------------------------------- #
148
149 @staticmethod
150 def is_Firewall_Blocked(resp):
151 try:
152 return (
153 resp.headers.get('Server', '').startswith('cloudflare')
154 and resp.status_code == 403
155 and re.search(
156 r'<span class="cf-error-code">1020</span>',
157 resp.text,
158 re.M | re.DOTALL
159 )
160 )
161 except AttributeError:
162 pass
163
164 return False
165
166 # ------------------------------------------------------------------------------- #
167 # Wrapper for is_Captcha_Challenge, is_IUAM_Challenge, is_Firewall_Blocked
168 # ------------------------------------------------------------------------------- #
169
170 def is_Challenge_Request(self, resp):
171 if self.is_Firewall_Blocked(resp):
172 self.cloudscraper.simpleException(
173 CloudflareCode1020,
174 'Cloudflare has blocked this request (Code 1020 Detected).'
175 )
176
177 if self.is_New_Captcha_Challenge(resp):
178 self.cloudscraper.simpleException(
179 CloudflareChallengeError,
180 'Detected a Cloudflare version 2 Captcha challenge, This feature is not available in the opensource (free) version.'
181 )
182
183 if self.is_New_IUAM_Challenge(resp):
184 self.cloudscraper.simpleException(
185 CloudflareChallengeError,
186 'Detected a Cloudflare version 2 challenge, This feature is not available in the opensource (free) version.'
187 )
188
189 if self.is_Captcha_Challenge(resp) or self.is_IUAM_Challenge(resp):
190 if self.cloudscraper.debug:
191 print('Detected a Cloudflare version 1 challenge.')
192 return True
193
194 return False
195
196 # ------------------------------------------------------------------------------- #
197 # Try to solve cloudflare javascript challenge.
198 # ------------------------------------------------------------------------------- #
199
200 def IUAM_Challenge_Response(self, body, url, interpreter):
201 try:
202 formPayload = re.search(
203 r'<form (?P<form>.*?="challenge-form" '
204 r'action="(?P<challengeUUID>.*?'
205 r'__cf_chl_f_tk=\S+)"(.*?)</form>)',
206 body,
207 re.M | re.DOTALL
208 ).groupdict()
209
210 if not all(key in formPayload for key in ['form', 'challengeUUID']):
211 self.cloudscraper.simpleException(
212 CloudflareIUAMError,
213 "Cloudflare IUAM detected, unfortunately we can't extract the parameters correctly."
214 )
215
216 payload = OrderedDict()
217 for challengeParam in re.findall(r'^\s*<input\s(.*?)/>', formPayload['form'], re.M | re.S):
218 inputPayload = dict(re.findall(r'(\S+)="(\S+)"', challengeParam))
219 if inputPayload.get('name') in ['r', 'jschl_vc', 'pass']:
220 payload.update({inputPayload['name']: inputPayload['value']})
221
222 except AttributeError:
223 self.cloudscraper.simpleException(
224 CloudflareIUAMError,
225 "Cloudflare IUAM detected, unfortunately we can't extract the parameters correctly."
226 )
227
228 hostParsed = urlparse(url)
229
230 try:
231 payload['jschl_answer'] = JavaScriptInterpreter.dynamicImport(
232 interpreter
233 ).solveChallenge(body, hostParsed.netloc)
234 except Exception as e:
235 self.cloudscraper.simpleException(
236 CloudflareIUAMError,
237 f"Unable to parse Cloudflare anti-bots page: {getattr(e, 'message', e)}"
238 )
239
240 return {
241 'url': f"{hostParsed.scheme}://{hostParsed.netloc}{self.unescape(formPayload['challengeUUID'])}",
242 'data': payload
243 }
244
245 # ------------------------------------------------------------------------------- #
246 # Try to solve the Captcha challenge via 3rd party.
247 # ------------------------------------------------------------------------------- #
248
249 def captcha_Challenge_Response(self, provider, provider_params, body, url):
250 try:
251 formPayload = re.search(
252 r'<form (?P<form>.*?="challenge-form" '
253 r'action="(?P<challengeUUID>.*?__cf_chl_captcha_tk__=\S+)"(.*?)</form>)',
254 body,
255 re.M | re.DOTALL
256 ).groupdict()
257
258 if not all(key in formPayload for key in ['form', 'challengeUUID']):
259 self.cloudscraper.simpleException(
260 CloudflareCaptchaError,
261 "Cloudflare Captcha detected, unfortunately we can't extract the parameters correctly."
262 )
263
264 payload = OrderedDict(
265 re.findall(
266 r'(name="r"\svalue|data-ray|data-sitekey|name="cf_captcha_kind"\svalue)="(.*?)"',
267 formPayload['form']
268 )
269 )
270
271 captchaType = 'reCaptcha' if payload['name="cf_captcha_kind" value'] == 're' else 'hCaptcha'
272
273 except (AttributeError, KeyError):
274 self.cloudscraper.simpleException(
275 CloudflareCaptchaError,
276 "Cloudflare Captcha detected, unfortunately we can't extract the parameters correctly."
277 )
278
279 # ------------------------------------------------------------------------------- #
280 # Pass proxy parameter to provider to solve captcha.
281 # ------------------------------------------------------------------------------- #
282
283 if self.cloudscraper.proxies and self.cloudscraper.proxies != self.cloudscraper.captcha.get('proxy'):
284 self.cloudscraper.captcha['proxy'] = self.proxies
285
286 # ------------------------------------------------------------------------------- #
287 # Pass User-Agent if provider supports it to solve captcha.
288 # ------------------------------------------------------------------------------- #
289
290 self.cloudscraper.captcha['User-Agent'] = self.cloudscraper.headers['User-Agent']
291
292 # ------------------------------------------------------------------------------- #
293 # Submit job to provider to request captcha solve.
294 # ------------------------------------------------------------------------------- #
295
296 captchaResponse = Captcha.dynamicImport(
297 provider.lower()
298 ).solveCaptcha(
299 captchaType,
300 url,
301 payload['data-sitekey'],
302 provider_params
303 )
304
305 # ------------------------------------------------------------------------------- #
306 # Parse and handle the response of solved captcha.
307 # ------------------------------------------------------------------------------- #
308
309 dataPayload = OrderedDict([
310 ('r', payload.get('name="r" value', '')),
311 ('cf_captcha_kind', payload['name="cf_captcha_kind" value']),
312 ('id', payload.get('data-ray')),
313 ('g-recaptcha-response', captchaResponse)
314 ])
315
316 if captchaType == 'hCaptcha':
317 dataPayload.update({'h-captcha-response': captchaResponse})
318
319 hostParsed = urlparse(url)
320
321 return {
322 'url': f"{hostParsed.scheme}://{hostParsed.netloc}{self.unescape(formPayload['challengeUUID'])}",
323 'data': dataPayload
324 }
325
326 # ------------------------------------------------------------------------------- #
327 # Attempt to handle and send the challenge response back to cloudflare
328 # ------------------------------------------------------------------------------- #
329
330 def Challenge_Response(self, resp, **kwargs):
331 if self.is_Captcha_Challenge(resp):
332 # ------------------------------------------------------------------------------- #
333 # double down on the request as some websites are only checking
334 # if cfuid is populated before issuing Captcha.
335 # ------------------------------------------------------------------------------- #
336
337 if self.cloudscraper.doubleDown:
338 resp = self.cloudscraper.decodeBrotli(
339 self.cloudscraper.perform_request(resp.request.method, resp.url, **kwargs)
340 )
341
342 if not self.is_Captcha_Challenge(resp):
343 return resp
344
345 # ------------------------------------------------------------------------------- #
346 # if no captcha provider raise a runtime error.
347 # ------------------------------------------------------------------------------- #
348
349 if (
350 not self.cloudscraper.captcha
351 or not isinstance(self.cloudscraper.captcha, dict)
352 or not self.cloudscraper.captcha.get('provider')
353 ):
354 self.cloudscraper.simpleException(
355 CloudflareCaptchaProvider,
356 "Cloudflare Captcha detected, unfortunately you haven't loaded an anti Captcha provider "
357 "correctly via the 'captcha' parameter."
358 )
359
360 # ------------------------------------------------------------------------------- #
361 # if provider is return_response, return the response without doing anything.
362 # ------------------------------------------------------------------------------- #
363
364 if self.cloudscraper.captcha.get('provider') == 'return_response':
365 return resp
366
367 # ------------------------------------------------------------------------------- #
368 # Submit request to parser wrapper to solve captcha
369 # ------------------------------------------------------------------------------- #
370
371 submit_url = self.captcha_Challenge_Response(
372 self.cloudscraper.captcha.get('provider'),
373 self.cloudscraper.captcha,
374 resp.text,
375 resp.url
376 )
377 else:
378 # ------------------------------------------------------------------------------- #
379 # Cloudflare requires a delay before solving the challenge
380 # ------------------------------------------------------------------------------- #
381
382 if not self.cloudscraper.delay:
383 try:
384 delay = float(
385 re.search(
386 r'submit\(\);\r?\n\s*},\s*([0-9]+)',
387 resp.text
388 ).group(1)
389 ) / float(1000)
390 if isinstance(delay, (int, float)):
391 self.cloudscraper.delay = delay
392 except (AttributeError, ValueError):
393 self.cloudscraper.simpleException(
394 CloudflareIUAMError,
395 "Cloudflare IUAM possibility malformed, issue extracing delay value."
396 )
397
398 time.sleep(self.cloudscraper.delay)
399
400 # ------------------------------------------------------------------------------- #
401
402 submit_url = self.IUAM_Challenge_Response(
403 resp.text,
404 resp.url,
405 self.cloudscraper.interpreter
406 )
407
408 # ------------------------------------------------------------------------------- #
409 # Send the Challenge Response back to Cloudflare
410 # ------------------------------------------------------------------------------- #
411
412 if submit_url:
413
414 def updateAttr(obj, name, newValue):
415 try:
416 obj[name].update(newValue)
417 return obj[name]
418 except (AttributeError, KeyError):
419 obj[name] = {}
420 obj[name].update(newValue)
421 return obj[name]
422
423 cloudflare_kwargs = deepcopy(kwargs)
424 cloudflare_kwargs['allow_redirects'] = False
425 cloudflare_kwargs['data'] = updateAttr(
426 cloudflare_kwargs,
427 'data',
428 submit_url['data']
429 )
430
431 urlParsed = urlparse(resp.url)
432 cloudflare_kwargs['headers'] = updateAttr(
433 cloudflare_kwargs,
434 'headers',
435 {
436 'Origin': f'{urlParsed.scheme}://{urlParsed.netloc}',
437 'Referer': resp.url
438 }
439 )
440
441 challengeSubmitResponse = self.cloudscraper.request(
442 'POST',
443 submit_url['url'],
444 **cloudflare_kwargs
445 )
446
447 if challengeSubmitResponse.status_code == 400:
448 self.cloudscraper.simpleException(
449 CloudflareSolveError,
450 'Invalid challenge answer detected, Cloudflare broken?'
451 )
452
453 # ------------------------------------------------------------------------------- #
454 # Return response if Cloudflare is doing content pass through instead of 3xx
455 # else request with redirect URL also handle protocol scheme change http -> https
456 # ------------------------------------------------------------------------------- #
457
458 if not challengeSubmitResponse.is_redirect:
459 return challengeSubmitResponse
460
461 else:
462 cloudflare_kwargs = deepcopy(kwargs)
463 cloudflare_kwargs['headers'] = updateAttr(
464 cloudflare_kwargs,
465 'headers',
466 {'Referer': challengeSubmitResponse.url}
467 )
468
469 if not urlparse(challengeSubmitResponse.headers['Location']).netloc:
470 redirect_location = urljoin(
471 challengeSubmitResponse.url,
472 challengeSubmitResponse.headers['Location']
473 )
474 else:
475 redirect_location = challengeSubmitResponse.headers['Location']
476
477 return self.cloudscraper.request(
478 resp.request.method,
479 redirect_location,
480 **cloudflare_kwargs
481 )
482
483 # ------------------------------------------------------------------------------- #
484 # We shouldn't be here...
485 # Re-request the original query and/or process again....
486 # ------------------------------------------------------------------------------- #
487
488 return self.cloudscraper.request(resp.request.method, resp.url, **kwargs)
489
490 # ------------------------------------------------------------------------------- #