Debian Codegraph @

tools/private_domains_checker/PSLPrivateDomainsProcessor.py

raw
1 import datetime
2 import json
3 import time
4
5 import pandas as pd
6 import requests
7 import whoisdomain as whois
8
9
10 def make_dns_request(domain, record_type):
11 """
12 Makes DNS requests to both Google and Cloudflare DNS APIs.
13
14 Args:
15 domain (str): The domain to query.
16 record_type (str): The type of DNS record to query.
17
18 Returns:
19 list: A list containing the JSON responses from Google and Cloudflare.
20 """
21 urls = [
22 f"https://dns.google/resolve?name={domain}&type={record_type}",
23 f"https://cloudflare-dns.com/dns-query?name={domain}&type={record_type}"
24 ]
25
26 headers = {
27 "accept": "application/dns-json"
28 }
29
30 responses = []
31 for url in urls:
32 try:
33 response = requests.get(url, headers=headers)
34 if response.status_code == 200:
35 json_response = response.json()
36 # print(f"URL: {url}, Response: {json_response}")
37 responses.append(json_response)
38 else:
39 # print(f"URL: {url}, Status Code: {response.status_code}")
40 responses.append(None)
41 except Exception as e:
42 print(f"URL: {url}, DNS Exception: {e}")
43 responses.append(None)
44
45 return responses
46
47
48 def check_dns_status(domain):
49 """
50 Checks the DNS status of a domain using Google and Cloudflare DNS APIs.
51
52 Args:
53 domain (str): The domain to check.
54
55 Returns:
56 str: The DNS status of the domain.
57 """
58
59 def make_request():
60 responses = make_dns_request(domain, "NS")
61 if None in responses:
62 return "ERROR"
63
64 google_status = responses[0].get("Status")
65 cloudflare_status = responses[1].get("Status")
66
67 print(f"Google Status: {google_status}, Cloudflare Status: {cloudflare_status}")
68
69 if google_status == cloudflare_status:
70 if google_status == 3:
71 return "NXDOMAIN"
72 else:
73 return "ok"
74 else:
75 return "INCONSISTENT"
76
77 for _ in range(5):
78 dns_status = make_request()
79 print(f"Attempt {_ + 1}, DNS Status: {dns_status}")
80 if dns_status not in ["ERROR", "INCONSISTENT"]:
81 return dns_status
82 time.sleep(1)
83 return "INCONSISTENT"
84
85
86 def check_psl_txt_record(domain):
87 """
88 Checks the _psl TXT record for a domain using Google and Cloudflare DNS APIs.
89
90 Args:
91 domain (str): The domain to check.
92
93 Returns:
94 str: The _psl TXT record status of the domain.
95 """
96 # Prepare the domain for the TXT check
97 domain = domain.lstrip('*.').lstrip('!').encode('idna').decode('ascii')
98
99 def make_request():
100 responses = make_dns_request(f"_psl.{domain}", "TXT")
101 if None in responses:
102 return "ERROR"
103
104 google_txt = responses[0].get("Answer", [])
105 cloudflare_txt = responses[1].get("Answer", [])
106
107 google_txt_records = [record.get("data", "") for record in google_txt]
108 cloudflare_txt_records = [record.get("data", "").strip('"') for record in cloudflare_txt]
109
110 print(
111 f"_psl TXT Records (Google): {google_txt_records}, _psl TXT Records (Cloudflare): {cloudflare_txt_records}")
112
113 if google_txt_records == cloudflare_txt_records:
114 for record in google_txt_records:
115 if "github.com/publicsuffix/list/pull/" in record:
116 return "valid"
117 return "invalid"
118 else:
119 return "INCONSISTENT"
120
121 for _ in range(5):
122 psl_txt_status = make_request()
123 print(f"Attempt {_ + 1}, PSL TXT Status: {psl_txt_status}")
124 if psl_txt_status not in ["ERROR", "INCONSISTENT"]:
125 return psl_txt_status
126 time.sleep(1)
127 return "INCONSISTENT"
128
129
130 def get_whois_data(domain):
131 """
132 Retrieves WHOIS data for a domain using the whoisdomain package.
133
134 Args:
135 domain (str): The domain to query.
136
137 Returns:
138 tuple: A tuple containing WHOIS domain status, expiry date, and WHOIS status.
139 """
140 try:
141 d = whois.query(domain)
142 whois_domain_status = d.statuses
143 whois_expiry = d.expiration_date
144 whois_status = "ok"
145 except Exception as e:
146 print(f"WHOIS Exception: {e}")
147 whois_domain_status = None
148 whois_expiry = None
149 whois_status = "ERROR"
150 return whois_domain_status, whois_expiry, whois_status
151
152
153 class PSLPrivateDomainsProcessor:
154 """
155 A class to process PSL private section domains, check their status, and save the results.
156 """
157
158 def __init__(self):
159 """
160 Initializes the PSLPrivateDomainsProcessor with default values and settings.
161 """
162 self.psl_url = "https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat"
163 self.psl_icann_marker = "// ===BEGIN ICANN DOMAINS==="
164 self.psl_private_marker = "// ===BEGIN PRIVATE DOMAINS==="
165 self.columns = [
166 "psl_entry",
167 "top_level_domain",
168 "dns_status",
169 "whois_status",
170 "whois_domain_expiry_date",
171 "whois_domain_status",
172 "psl_txt_status",
173 "expiry_check_status"
174 ]
175 self.df = pd.DataFrame(columns=self.columns)
176 self.icann_domains = set()
177
178 def fetch_psl_data(self):
179 """
180 Fetches the PSL data from the specified URL.
181
182 Returns:
183 str: The raw PSL data.
184 """
185 print("Fetching PSL data from URL...")
186 response = requests.get(self.psl_url)
187 psl_data = response.text
188 print("PSL data fetched.")
189 return psl_data
190
191 def parse_domain(self, domain):
192 """
193 Parses and normalizes a domain.
194
195 Args:
196 domain (str): The domain to parse.
197
198 Returns:
199 str: The normalized domain.
200
201 Raises:
202 ValueError: If no valid top-level domain is found.
203 """
204 domain = domain.lstrip('*.') # wildcards (*)
205 domain = domain.lstrip('!') # bangs (!)
206
207 parts = domain.split('.')
208
209 for i in range(len(parts)):
210 candidate = '.'.join(parts[i:])
211 if candidate in self.icann_domains:
212 continue
213 elif '.'.join(parts[i + 1:]) in self.icann_domains:
214 return candidate.encode('idna').decode('ascii')
215
216 raise ValueError(f"No valid top-level domain found in the provided domain: {domain}")
217
218 def parse_psl_data(self, psl_data):
219 """
220 Parses the fetched PSL data and separates ICANN and private domains.
221
222 Args:
223 psl_data (str): The raw PSL data.
224
225 Returns:
226 tuple: A tuple containing the unparsed private domains and the parsed private domains.
227 """
228 print("Parsing PSL data...")
229
230 lines = psl_data.splitlines()
231 process_icann = False
232 process_private = False
233 raw_private_domains = []
234 parsed_private_domains = []
235
236 for line in lines:
237 stripped_line = line.strip()
238 if stripped_line == self.psl_icann_marker:
239 process_icann = True
240 process_private = False
241 continue
242 elif stripped_line == self.psl_private_marker:
243 process_icann = False
244 process_private = True
245 continue
246
247 if stripped_line.startswith('//') or not stripped_line:
248 continue
249
250 if process_icann:
251 self.icann_domains.add(stripped_line)
252 elif process_private:
253 raw_private_domains.append(stripped_line)
254 parsed_private_domains.append(stripped_line)
255
256 print(f"Private domains to be processed: {len(parsed_private_domains)}\n"
257 f"ICANN domains: {len(self.icann_domains)}")
258
259 parsed_private_domains = [self.parse_domain(domain) for domain in parsed_private_domains]
260 raw_private_domains = list(set(raw_private_domains))
261 parsed_private_domains = list(set(parsed_private_domains))
262 print("Private domains in the publicly registrable name space: ", len(parsed_private_domains))
263
264 return raw_private_domains, parsed_private_domains
265
266 def process_domains(self, raw_domains, domains):
267 """
268 Processes each domain, performing DNS, WHOIS, and _psl TXT record checks.
269
270 Args:
271 raw_domains (list): A list of unparsed domains to process.
272 domains (list): A list of domains to process.
273 """
274 data = []
275 for raw_domain, domain in zip(raw_domains, domains):
276 whois_domain_status, whois_expiry, whois_status = get_whois_data(domain)
277 dns_status = check_dns_status(domain)
278 psl_txt_status = check_psl_txt_record(raw_domain)
279
280 if whois_status == "ERROR":
281 expiry_check_status = "ERROR"
282 else:
283 expiry_check_status = "ok" if whois_expiry and whois_expiry >= (
284 datetime.datetime.utcnow() + datetime.timedelta(days=365 * 2)) else "FAIL_2Y"
285
286 print(
287 f"{domain} - DNS Status: {dns_status}, Expiry: {whois_expiry}, "
288 f"PSL TXT Status: {psl_txt_status}, Expiry Check: {expiry_check_status}")
289
290 data.append({
291 "psl_entry": domain,
292 "top_level_domain": domain,
293 "whois_domain_status": json.dumps(whois_domain_status),
294 "whois_domain_expiry_date": whois_expiry,
295 "whois_status": whois_status,
296 "dns_status": dns_status,
297 "psl_txt_status": psl_txt_status,
298 "expiry_check_status": expiry_check_status
299 })
300
301 self.df = pd.DataFrame(data, columns=self.columns)
302
303 def save_results(self):
304 """
305 Saves all processed domain data to data/all.csv.
306 """
307 sorted_df = self.df.sort_values(by="psl_entry")
308 sorted_df.to_csv("data/all.csv", index=False)
309
310 def save_invalid_results(self):
311 """
312 Saves domains with invalid DNS or expired WHOIS data to data/nxdomain.csv and data/expired.csv.
313 """
314 nxdomain_df = self.df[self.df["dns_status"] != "ok"].sort_values(by="psl_entry")
315 nxdomain_df.to_csv("data/nxdomain.csv", index=False)
316
317 today_str = datetime.datetime.utcnow().strftime("%Y-%m-%d")
318 expired_df = self.df[
319 self.df["whois_domain_expiry_date"].notnull() &
320 (self.df["whois_domain_expiry_date"].astype(str).str[:10] < today_str)
321 ].sort_values(by="psl_entry")
322 expired_df.to_csv("data/expired.csv", index=False)
323
324 def save_hold_results(self):
325 """
326 Saves domains with WHOIS status containing any form of "hold" to data/hold.csv.
327 """
328 hold_df = self.df[
329 self.df["whois_domain_status"].str.contains("hold", case=False, na=False)
330 ].sort_values(by="psl_entry")
331 hold_df.to_csv("data/hold.csv", index=False)
332
333 def save_missing_psl_txt_results(self):
334 """
335 Saves domains with invalid _psl TXT records to data/missing_psl_txt.csv.
336 """
337 missing_psl_txt_df = self.df[self.df["psl_txt_status"] == "invalid"].sort_values(by="psl_entry")
338 missing_psl_txt_df.to_csv("data/missing_psl_txt.csv", index=False)
339
340 def save_expiry_less_than_2yrs_results(self):
341 """
342 Saves domains with WHOIS expiry date less than 2 years from now to data/expiry_less_than_2yrs.csv.
343 """
344 expiry_less_than_2yrs_df = self.df[self.df["expiry_check_status"] == "FAIL_2Y"].sort_values(by="psl_entry")
345 expiry_less_than_2yrs_df.to_csv("data/expiry_less_than_2yrs.csv", index=False)
346
347 def run(self):
348 """
349 Executes the entire processing pipeline.
350 """
351 psl_data = self.fetch_psl_data()
352 raw_domains, domains = self.parse_psl_data(psl_data)
353 self.process_domains(raw_domains, domains)
354 self.save_results()
355 self.save_invalid_results()
356 self.save_hold_results()
357 self.save_missing_psl_txt_results()
358 self.save_expiry_less_than_2yrs_results()
359
360
361 if __name__ == "__main__":
362 processor = PSLPrivateDomainsProcessor()
363 processor.run()