Debian Codegraph @

cloudscraper/__init__.py

raw
1 # ------------------------------------------------------------------------------- #
2
3 import logging
4 import requests
5 import sys
6 import ssl
7
8 from requests.adapters import HTTPAdapter
9 from requests.sessions import Session
10 from requests_toolbelt.utils import dump
11
12 # ------------------------------------------------------------------------------- #
13
14 try:
15 import brotli
16 except ImportError:
17 pass
18
19 try:
20 import copyreg
21 except ImportError:
22 import copy_reg as copyreg
23
24 try:
25 from urlparse import urlparse
26 except ImportError:
27 from urllib.parse import urlparse
28
29 # ------------------------------------------------------------------------------- #
30
31 from .exceptions import (
32 CloudflareLoopProtection,
33 CloudflareIUAMError
34 )
35
36 from .cloudflare import Cloudflare
37 from .user_agent import User_Agent
38
39 # ------------------------------------------------------------------------------- #
40
41 __version__ = '1.2.71'
42
43 # ------------------------------------------------------------------------------- #
44
45
46 class CipherSuiteAdapter(HTTPAdapter):
47
48 __attrs__ = [
49 'ssl_context',
50 'max_retries',
51 'config',
52 '_pool_connections',
53 '_pool_maxsize',
54 '_pool_block',
55 'source_address'
56 ]
57
58 def __init__(self, *args, **kwargs):
59 self.ssl_context = kwargs.pop('ssl_context', None)
60 self.cipherSuite = kwargs.pop('cipherSuite', None)
61 self.source_address = kwargs.pop('source_address', None)
62 self.server_hostname = kwargs.pop('server_hostname', None)
63 self.ecdhCurve = kwargs.pop('ecdhCurve', 'prime256v1')
64
65 if self.source_address:
66 if isinstance(self.source_address, str):
67 self.source_address = (self.source_address, 0)
68
69 if not isinstance(self.source_address, tuple):
70 raise TypeError(
71 "source_address must be IP address string or (ip, port) tuple"
72 )
73
74 if not self.ssl_context:
75 self.ssl_context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
76
77 self.ssl_context.orig_wrap_socket = self.ssl_context.wrap_socket
78 self.ssl_context.wrap_socket = self.wrap_socket
79
80 if self.server_hostname:
81 self.ssl_context.server_hostname = self.server_hostname
82
83 self.ssl_context.set_ciphers(self.cipherSuite)
84 self.ssl_context.set_ecdh_curve(self.ecdhCurve)
85
86 self.ssl_context.minimum_version = ssl.TLSVersion.TLSv1_2
87 self.ssl_context.maximum_version = ssl.TLSVersion.TLSv1_3
88
89 super(CipherSuiteAdapter, self).__init__(**kwargs)
90
91 # ------------------------------------------------------------------------------- #
92
93 def wrap_socket(self, *args, **kwargs):
94 if hasattr(self.ssl_context, 'server_hostname') and self.ssl_context.server_hostname:
95 kwargs['server_hostname'] = self.ssl_context.server_hostname
96 self.ssl_context.check_hostname = False
97 else:
98 self.ssl_context.check_hostname = True
99
100 return self.ssl_context.orig_wrap_socket(*args, **kwargs)
101
102 # ------------------------------------------------------------------------------- #
103
104 def init_poolmanager(self, *args, **kwargs):
105 kwargs['ssl_context'] = self.ssl_context
106 kwargs['source_address'] = self.source_address
107 return super(CipherSuiteAdapter, self).init_poolmanager(*args, **kwargs)
108
109 # ------------------------------------------------------------------------------- #
110
111 def proxy_manager_for(self, *args, **kwargs):
112 kwargs['ssl_context'] = self.ssl_context
113 kwargs['source_address'] = self.source_address
114 return super(CipherSuiteAdapter, self).proxy_manager_for(*args, **kwargs)
115
116 # ------------------------------------------------------------------------------- #
117
118
119 class CloudScraper(Session):
120
121 def __init__(self, *args, **kwargs):
122 self.debug = kwargs.pop('debug', False)
123
124 self.disableCloudflareV1 = kwargs.pop('disableCloudflareV1', False)
125 self.delay = kwargs.pop('delay', None)
126 self.captcha = kwargs.pop('captcha', {})
127 self.doubleDown = kwargs.pop('doubleDown', True)
128 self.interpreter = kwargs.pop('interpreter', 'native')
129
130 self.requestPreHook = kwargs.pop('requestPreHook', None)
131 self.requestPostHook = kwargs.pop('requestPostHook', None)
132
133 self.cipherSuite = kwargs.pop('cipherSuite', None)
134 self.ecdhCurve = kwargs.pop('ecdhCurve', 'prime256v1')
135 self.source_address = kwargs.pop('source_address', None)
136 self.server_hostname = kwargs.pop('server_hostname', None)
137 self.ssl_context = kwargs.pop('ssl_context', None)
138
139 self.allow_brotli = kwargs.pop(
140 'allow_brotli',
141 True if 'brotli' in sys.modules.keys() else False
142 )
143
144 self.user_agent = User_Agent(
145 allow_brotli=self.allow_brotli,
146 browser=kwargs.pop('browser', None)
147 )
148
149 self._solveDepthCnt = 0
150 self.solveDepth = kwargs.pop('solveDepth', 3)
151
152 super(CloudScraper, self).__init__(*args, **kwargs)
153
154 # pylint: disable=E0203
155 if 'requests' in self.headers['User-Agent']:
156 # ------------------------------------------------------------------------------- #
157 # Set a random User-Agent if no custom User-Agent has been set
158 # ------------------------------------------------------------------------------- #
159 self.headers = self.user_agent.headers
160 if not self.cipherSuite:
161 self.cipherSuite = self.user_agent.cipherSuite
162
163 if isinstance(self.cipherSuite, list):
164 self.cipherSuite = ':'.join(self.cipherSuite)
165
166 self.mount(
167 'https://',
168 CipherSuiteAdapter(
169 cipherSuite=self.cipherSuite,
170 ecdhCurve=self.ecdhCurve,
171 server_hostname=self.server_hostname,
172 source_address=self.source_address,
173 ssl_context=self.ssl_context
174 )
175 )
176
177 # purely to allow us to pickle dump
178 copyreg.pickle(ssl.SSLContext, lambda obj: (obj.__class__, (obj.protocol,)))
179
180 # ------------------------------------------------------------------------------- #
181 # Allow us to pickle our session back with all variables
182 # ------------------------------------------------------------------------------- #
183
184 def __getstate__(self):
185 return self.__dict__
186
187 # ------------------------------------------------------------------------------- #
188 # Allow replacing actual web request call via subclassing
189 # ------------------------------------------------------------------------------- #
190
191 def perform_request(self, method, url, *args, **kwargs):
192 return super(CloudScraper, self).request(method, url, *args, **kwargs)
193
194 # ------------------------------------------------------------------------------- #
195 # Raise an Exception with no stacktrace and reset depth counter.
196 # ------------------------------------------------------------------------------- #
197
198 def simpleException(self, exception, msg):
199 self._solveDepthCnt = 0
200 sys.tracebacklimit = 0
201 raise exception(msg)
202
203 # ------------------------------------------------------------------------------- #
204 # debug the request via the response
205 # ------------------------------------------------------------------------------- #
206
207 @staticmethod
208 def debugRequest(req):
209 try:
210 print(dump.dump_all(req).decode('utf-8', errors='backslashreplace'))
211 except ValueError as e:
212 print(f"Debug Error: {getattr(e, 'message', e)}")
213
214 # ------------------------------------------------------------------------------- #
215 # Decode Brotli on older versions of urllib3 manually
216 # ------------------------------------------------------------------------------- #
217
218 def decodeBrotli(self, resp):
219 if requests.packages.urllib3.__version__ < '1.25.1' and resp.headers.get('Content-Encoding') == 'br':
220 if self.allow_brotli and resp._content:
221 resp._content = brotli.decompress(resp.content)
222 else:
223 logging.warning(
224 f'You\'re running urllib3 {requests.packages.urllib3.__version__}, Brotli content detected, '
225 'Which requires manual decompression, '
226 'But option allow_brotli is set to False, '
227 'We will not continue to decompress.'
228 )
229
230 return resp
231
232 # ------------------------------------------------------------------------------- #
233 # Our hijacker request function
234 # ------------------------------------------------------------------------------- #
235
236 def request(self, method, url, *args, **kwargs):
237 # pylint: disable=E0203
238 if kwargs.get('proxies') and kwargs.get('proxies') != self.proxies:
239 self.proxies = kwargs.get('proxies')
240
241 # ------------------------------------------------------------------------------- #
242 # Pre-Hook the request via user defined function.
243 # ------------------------------------------------------------------------------- #
244
245 if self.requestPreHook:
246 (method, url, args, kwargs) = self.requestPreHook(
247 self,
248 method,
249 url,
250 *args,
251 **kwargs
252 )
253
254 # ------------------------------------------------------------------------------- #
255 # Make the request via requests.
256 # ------------------------------------------------------------------------------- #
257
258 response = self.decodeBrotli(
259 self.perform_request(method, url, *args, **kwargs)
260 )
261
262 # ------------------------------------------------------------------------------- #
263 # Debug the request via the Response object.
264 # ------------------------------------------------------------------------------- #
265
266 if self.debug:
267 self.debugRequest(response)
268
269 # ------------------------------------------------------------------------------- #
270 # Post-Hook the request aka Post-Hook the response via user defined function.
271 # ------------------------------------------------------------------------------- #
272
273 if self.requestPostHook:
274 newResponse = self.requestPostHook(self, response)
275
276 if response != newResponse: # Give me walrus in 3.7!!!
277 response = newResponse
278 if self.debug:
279 print('==== requestPostHook Debug ====')
280 self.debugRequest(response)
281
282 # ------------------------------------------------------------------------------- #
283
284 if not self.disableCloudflareV1:
285 cloudflareV1 = Cloudflare(self)
286
287 # ------------------------------------------------------------------------------- #
288 # Check if Cloudflare v1 anti-bot is on
289 # ------------------------------------------------------------------------------- #
290
291 if cloudflareV1.is_Challenge_Request(response):
292 # ------------------------------------------------------------------------------- #
293 # Try to solve the challenge and send it back
294 # ------------------------------------------------------------------------------- #
295
296 if self._solveDepthCnt >= self.solveDepth:
297 _ = self._solveDepthCnt
298 self.simpleException(
299 CloudflareLoopProtection,
300 f"!!Loop Protection!! We have tried to solve {_} time(s) in a row."
301 )
302
303 self._solveDepthCnt += 1
304
305 response = cloudflareV1.Challenge_Response(response, **kwargs)
306 else:
307 if not response.is_redirect and response.status_code not in [429, 503]:
308 self._solveDepthCnt = 0
309
310 return response
311
312 # ------------------------------------------------------------------------------- #
313
314 @classmethod
315 def create_scraper(cls, sess=None, **kwargs):
316 """
317 Convenience function for creating a ready-to-go CloudScraper object.
318 """
319 scraper = cls(**kwargs)
320
321 if sess:
322 for attr in ['auth', 'cert', 'cookies', 'headers', 'hooks', 'params', 'proxies', 'data']:
323 val = getattr(sess, attr, None)
324 if val is not None:
325 setattr(scraper, attr, val)
326
327 return scraper
328
329 # ------------------------------------------------------------------------------- #
330 # Functions for integrating cloudscraper with other applications and scripts
331 # ------------------------------------------------------------------------------- #
332
333 @classmethod
334 def get_tokens(cls, url, **kwargs):
335 scraper = cls.create_scraper(
336 **{
337 field: kwargs.pop(field, None) for field in [
338 'allow_brotli',
339 'browser',
340 'debug',
341 'delay',
342 'doubleDown',
343 'captcha',
344 'interpreter',
345 'source_address',
346 'requestPreHook',
347 'requestPostHook'
348 ] if field in kwargs
349 }
350 )
351
352 try:
353 resp = scraper.get(url, **kwargs)
354 resp.raise_for_status()
355 except Exception:
356 logging.error(f'"{url}" returned an error. Could not collect tokens.')
357 raise
358
359 domain = urlparse(resp.url).netloc
360 # noinspection PyUnusedLocal
361 cookie_domain = None
362
363 for d in scraper.cookies.list_domains():
364 if d.startswith('.') and d in (f'.{domain}'):
365 cookie_domain = d
366 break
367 else:
368 cls.simpleException(
369 cls,
370 CloudflareIUAMError,
371 "Unable to find Cloudflare cookies. Does the site actually "
372 "have Cloudflare IUAM (I'm Under Attack Mode) enabled?"
373 )
374
375 return (
376 {
377 'cf_clearance': scraper.cookies.get('cf_clearance', '', domain=cookie_domain)
378 },
379 scraper.headers['User-Agent']
380 )
381
382 # ------------------------------------------------------------------------------- #
383
384 @classmethod
385 def get_cookie_string(cls, url, **kwargs):
386 """
387 Convenience function for building a Cookie HTTP header value.
388 """
389 tokens, user_agent = cls.get_tokens(url, **kwargs)
390 return '; '.join('='.join(pair) for pair in tokens.items()), user_agent
391
392
393 # ------------------------------------------------------------------------------- #
394
395 if ssl.OPENSSL_VERSION_INFO < (1, 1, 1):
396 print(
397 f"DEPRECATION: The OpenSSL being used by this python install ({ssl.OPENSSL_VERSION}) does not meet the minimum supported "
398 "version (>= OpenSSL 1.1.1) in order to support TLS 1.3 required by Cloudflare, "
399 "You may encounter an unexpected Captcha or cloudflare 1020 blocks."
400 )
401
402 # ------------------------------------------------------------------------------- #
403
404 create_scraper = CloudScraper.create_scraper
405 session = CloudScraper.create_scraper
406 get_tokens = CloudScraper.get_tokens
407 get_cookie_string = CloudScraper.get_cookie_string