| 1 |
import datetime |
| 2 |
import json |
| 3 |
import time |
| 4 |
|
| 5 |
import pandas as pd |
| 6 |
import requests |
| 7 |
import whoisdomain as whois |
| 8 |
|
| 9 |
|
| 10 |
def make_dns_request(domain, record_type): |
| 11 |
""" |
| 12 |
Makes DNS requests to both Google and Cloudflare DNS APIs. |
| 13 |
|
| 14 |
Args: |
| 15 |
domain (str): The domain to query. |
| 16 |
record_type (str): The type of DNS record to query. |
| 17 |
|
| 18 |
Returns: |
| 19 |
list: A list containing the JSON responses from Google and Cloudflare. |
| 20 |
""" |
| 21 |
urls = [ |
| 22 |
f"https://dns.google/resolve?name={domain}&type={record_type}", |
| 23 |
f"https://cloudflare-dns.com/dns-query?name={domain}&type={record_type}" |
| 24 |
] |
| 25 |
|
| 26 |
headers = { |
| 27 |
"accept": "application/dns-json" |
| 28 |
} |
| 29 |
|
| 30 |
responses = [] |
| 31 |
for url in urls: |
| 32 |
try: |
| 33 |
response = requests.get(url, headers=headers) |
| 34 |
if response.status_code == 200: |
| 35 |
json_response = response.json() |
| 36 |
# print(f"URL: {url}, Response: {json_response}") |
| 37 |
responses.append(json_response) |
| 38 |
else: |
| 39 |
# print(f"URL: {url}, Status Code: {response.status_code}") |
| 40 |
responses.append(None) |
| 41 |
except Exception as e: |
| 42 |
print(f"URL: {url}, DNS Exception: {e}") |
| 43 |
responses.append(None) |
| 44 |
|
| 45 |
return responses |
| 46 |
|
| 47 |
|
| 48 |
def check_dns_status(domain): |
| 49 |
""" |
| 50 |
Checks the DNS status of a domain using Google and Cloudflare DNS APIs. |
| 51 |
|
| 52 |
Args: |
| 53 |
domain (str): The domain to check. |
| 54 |
|
| 55 |
Returns: |
| 56 |
str: The DNS status of the domain. |
| 57 |
""" |
| 58 |
|
| 59 |
def make_request(): |
| 60 |
responses = make_dns_request(domain, "NS") |
| 61 |
if None in responses: |
| 62 |
return "ERROR" |
| 63 |
|
| 64 |
google_status = responses[0].get("Status") |
| 65 |
cloudflare_status = responses[1].get("Status") |
| 66 |
|
| 67 |
print(f"Google Status: {google_status}, Cloudflare Status: {cloudflare_status}") |
| 68 |
|
| 69 |
if google_status == cloudflare_status: |
| 70 |
if google_status == 3: |
| 71 |
return "NXDOMAIN" |
| 72 |
else: |
| 73 |
return "ok" |
| 74 |
else: |
| 75 |
return "INCONSISTENT" |
| 76 |
|
| 77 |
for _ in range(5): |
| 78 |
dns_status = make_request() |
| 79 |
print(f"Attempt {_ + 1}, DNS Status: {dns_status}") |
| 80 |
if dns_status not in ["ERROR", "INCONSISTENT"]: |
| 81 |
return dns_status |
| 82 |
time.sleep(1) |
| 83 |
return "INCONSISTENT" |
| 84 |
|
| 85 |
|
| 86 |
def check_psl_txt_record(domain): |
| 87 |
""" |
| 88 |
Checks the _psl TXT record for a domain using Google and Cloudflare DNS APIs. |
| 89 |
|
| 90 |
Args: |
| 91 |
domain (str): The domain to check. |
| 92 |
|
| 93 |
Returns: |
| 94 |
str: The _psl TXT record status of the domain. |
| 95 |
""" |
| 96 |
# Prepare the domain for the TXT check |
| 97 |
domain = domain.lstrip('*.').lstrip('!').encode('idna').decode('ascii') |
| 98 |
|
| 99 |
def make_request(): |
| 100 |
responses = make_dns_request(f"_psl.{domain}", "TXT") |
| 101 |
if None in responses: |
| 102 |
return "ERROR" |
| 103 |
|
| 104 |
google_txt = responses[0].get("Answer", []) |
| 105 |
cloudflare_txt = responses[1].get("Answer", []) |
| 106 |
|
| 107 |
google_txt_records = [record.get("data", "") for record in google_txt] |
| 108 |
cloudflare_txt_records = [record.get("data", "").strip('"') for record in cloudflare_txt] |
| 109 |
|
| 110 |
print( |
| 111 |
f"_psl TXT Records (Google): {google_txt_records}, _psl TXT Records (Cloudflare): {cloudflare_txt_records}") |
| 112 |
|
| 113 |
if google_txt_records == cloudflare_txt_records: |
| 114 |
for record in google_txt_records: |
| 115 |
if "github.com/publicsuffix/list/pull/" in record: |
| 116 |
return "valid" |
| 117 |
return "invalid" |
| 118 |
else: |
| 119 |
return "INCONSISTENT" |
| 120 |
|
| 121 |
for _ in range(5): |
| 122 |
psl_txt_status = make_request() |
| 123 |
print(f"Attempt {_ + 1}, PSL TXT Status: {psl_txt_status}") |
| 124 |
if psl_txt_status not in ["ERROR", "INCONSISTENT"]: |
| 125 |
return psl_txt_status |
| 126 |
time.sleep(1) |
| 127 |
return "INCONSISTENT" |
| 128 |
|
| 129 |
|
| 130 |
def get_whois_data(domain): |
| 131 |
""" |
| 132 |
Retrieves WHOIS data for a domain using the whoisdomain package. |
| 133 |
|
| 134 |
Args: |
| 135 |
domain (str): The domain to query. |
| 136 |
|
| 137 |
Returns: |
| 138 |
tuple: A tuple containing WHOIS domain status, expiry date, and WHOIS status. |
| 139 |
""" |
| 140 |
try: |
| 141 |
d = whois.query(domain) |
| 142 |
whois_domain_status = d.statuses |
| 143 |
whois_expiry = d.expiration_date |
| 144 |
whois_status = "ok" |
| 145 |
except Exception as e: |
| 146 |
print(f"WHOIS Exception: {e}") |
| 147 |
whois_domain_status = None |
| 148 |
whois_expiry = None |
| 149 |
whois_status = "ERROR" |
| 150 |
return whois_domain_status, whois_expiry, whois_status |
| 151 |
|
| 152 |
|
| 153 |
class PSLPrivateDomainsProcessor: |
| 154 |
""" |
| 155 |
A class to process PSL private section domains, check their status, and save the results. |
| 156 |
""" |
| 157 |
|
| 158 |
def __init__(self): |
| 159 |
""" |
| 160 |
Initializes the PSLPrivateDomainsProcessor with default values and settings. |
| 161 |
""" |
| 162 |
self.psl_url = "https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat" |
| 163 |
self.psl_icann_marker = "// ===BEGIN ICANN DOMAINS===" |
| 164 |
self.psl_private_marker = "// ===BEGIN PRIVATE DOMAINS===" |
| 165 |
self.columns = [ |
| 166 |
"psl_entry", |
| 167 |
"top_level_domain", |
| 168 |
"dns_status", |
| 169 |
"whois_status", |
| 170 |
"whois_domain_expiry_date", |
| 171 |
"whois_domain_status", |
| 172 |
"psl_txt_status", |
| 173 |
"expiry_check_status" |
| 174 |
] |
| 175 |
self.df = pd.DataFrame(columns=self.columns) |
| 176 |
self.icann_domains = set() |
| 177 |
|
| 178 |
def fetch_psl_data(self): |
| 179 |
""" |
| 180 |
Fetches the PSL data from the specified URL. |
| 181 |
|
| 182 |
Returns: |
| 183 |
str: The raw PSL data. |
| 184 |
""" |
| 185 |
print("Fetching PSL data from URL...") |
| 186 |
response = requests.get(self.psl_url) |
| 187 |
psl_data = response.text |
| 188 |
print("PSL data fetched.") |
| 189 |
return psl_data |
| 190 |
|
| 191 |
def parse_domain(self, domain): |
| 192 |
""" |
| 193 |
Parses and normalizes a domain. |
| 194 |
|
| 195 |
Args: |
| 196 |
domain (str): The domain to parse. |
| 197 |
|
| 198 |
Returns: |
| 199 |
str: The normalized domain. |
| 200 |
|
| 201 |
Raises: |
| 202 |
ValueError: If no valid top-level domain is found. |
| 203 |
""" |
| 204 |
domain = domain.lstrip('*.') # wildcards (*) |
| 205 |
domain = domain.lstrip('!') # bangs (!) |
| 206 |
|
| 207 |
parts = domain.split('.') |
| 208 |
|
| 209 |
for i in range(len(parts)): |
| 210 |
candidate = '.'.join(parts[i:]) |
| 211 |
if candidate in self.icann_domains: |
| 212 |
continue |
| 213 |
elif '.'.join(parts[i + 1:]) in self.icann_domains: |
| 214 |
return candidate.encode('idna').decode('ascii') |
| 215 |
|
| 216 |
raise ValueError(f"No valid top-level domain found in the provided domain: {domain}") |
| 217 |
|
| 218 |
def parse_psl_data(self, psl_data): |
| 219 |
""" |
| 220 |
Parses the fetched PSL data and separates ICANN and private domains. |
| 221 |
|
| 222 |
Args: |
| 223 |
psl_data (str): The raw PSL data. |
| 224 |
|
| 225 |
Returns: |
| 226 |
tuple: A tuple containing the unparsed private domains and the parsed private domains. |
| 227 |
""" |
| 228 |
print("Parsing PSL data...") |
| 229 |
|
| 230 |
lines = psl_data.splitlines() |
| 231 |
process_icann = False |
| 232 |
process_private = False |
| 233 |
raw_private_domains = [] |
| 234 |
parsed_private_domains = [] |
| 235 |
|
| 236 |
for line in lines: |
| 237 |
stripped_line = line.strip() |
| 238 |
if stripped_line == self.psl_icann_marker: |
| 239 |
process_icann = True |
| 240 |
process_private = False |
| 241 |
continue |
| 242 |
elif stripped_line == self.psl_private_marker: |
| 243 |
process_icann = False |
| 244 |
process_private = True |
| 245 |
continue |
| 246 |
|
| 247 |
if stripped_line.startswith('//') or not stripped_line: |
| 248 |
continue |
| 249 |
|
| 250 |
if process_icann: |
| 251 |
self.icann_domains.add(stripped_line) |
| 252 |
elif process_private: |
| 253 |
raw_private_domains.append(stripped_line) |
| 254 |
parsed_private_domains.append(stripped_line) |
| 255 |
|
| 256 |
print(f"Private domains to be processed: {len(parsed_private_domains)}\n" |
| 257 |
f"ICANN domains: {len(self.icann_domains)}") |
| 258 |
|
| 259 |
parsed_private_domains = [self.parse_domain(domain) for domain in parsed_private_domains] |
| 260 |
raw_private_domains = list(set(raw_private_domains)) |
| 261 |
parsed_private_domains = list(set(parsed_private_domains)) |
| 262 |
print("Private domains in the publicly registrable name space: ", len(parsed_private_domains)) |
| 263 |
|
| 264 |
return raw_private_domains, parsed_private_domains |
| 265 |
|
| 266 |
def process_domains(self, raw_domains, domains): |
| 267 |
""" |
| 268 |
Processes each domain, performing DNS, WHOIS, and _psl TXT record checks. |
| 269 |
|
| 270 |
Args: |
| 271 |
raw_domains (list): A list of unparsed domains to process. |
| 272 |
domains (list): A list of domains to process. |
| 273 |
""" |
| 274 |
data = [] |
| 275 |
for raw_domain, domain in zip(raw_domains, domains): |
| 276 |
whois_domain_status, whois_expiry, whois_status = get_whois_data(domain) |
| 277 |
dns_status = check_dns_status(domain) |
| 278 |
psl_txt_status = check_psl_txt_record(raw_domain) |
| 279 |
|
| 280 |
if whois_status == "ERROR": |
| 281 |
expiry_check_status = "ERROR" |
| 282 |
else: |
| 283 |
expiry_check_status = "ok" if whois_expiry and whois_expiry >= ( |
| 284 |
datetime.datetime.utcnow() + datetime.timedelta(days=365 * 2)) else "FAIL_2Y" |
| 285 |
|
| 286 |
print( |
| 287 |
f"{domain} - DNS Status: {dns_status}, Expiry: {whois_expiry}, " |
| 288 |
f"PSL TXT Status: {psl_txt_status}, Expiry Check: {expiry_check_status}") |
| 289 |
|
| 290 |
data.append({ |
| 291 |
"psl_entry": domain, |
| 292 |
"top_level_domain": domain, |
| 293 |
"whois_domain_status": json.dumps(whois_domain_status), |
| 294 |
"whois_domain_expiry_date": whois_expiry, |
| 295 |
"whois_status": whois_status, |
| 296 |
"dns_status": dns_status, |
| 297 |
"psl_txt_status": psl_txt_status, |
| 298 |
"expiry_check_status": expiry_check_status |
| 299 |
}) |
| 300 |
|
| 301 |
self.df = pd.DataFrame(data, columns=self.columns) |
| 302 |
|
| 303 |
def save_results(self): |
| 304 |
""" |
| 305 |
Saves all processed domain data to data/all.csv. |
| 306 |
""" |
| 307 |
sorted_df = self.df.sort_values(by="psl_entry") |
| 308 |
sorted_df.to_csv("data/all.csv", index=False) |
| 309 |
|
| 310 |
def save_invalid_results(self): |
| 311 |
""" |
| 312 |
Saves domains with invalid DNS or expired WHOIS data to data/nxdomain.csv and data/expired.csv. |
| 313 |
""" |
| 314 |
nxdomain_df = self.df[self.df["dns_status"] != "ok"].sort_values(by="psl_entry") |
| 315 |
nxdomain_df.to_csv("data/nxdomain.csv", index=False) |
| 316 |
|
| 317 |
today_str = datetime.datetime.utcnow().strftime("%Y-%m-%d") |
| 318 |
expired_df = self.df[ |
| 319 |
self.df["whois_domain_expiry_date"].notnull() & |
| 320 |
(self.df["whois_domain_expiry_date"].astype(str).str[:10] < today_str) |
| 321 |
].sort_values(by="psl_entry") |
| 322 |
expired_df.to_csv("data/expired.csv", index=False) |
| 323 |
|
| 324 |
def save_hold_results(self): |
| 325 |
""" |
| 326 |
Saves domains with WHOIS status containing any form of "hold" to data/hold.csv. |
| 327 |
""" |
| 328 |
hold_df = self.df[ |
| 329 |
self.df["whois_domain_status"].str.contains("hold", case=False, na=False) |
| 330 |
].sort_values(by="psl_entry") |
| 331 |
hold_df.to_csv("data/hold.csv", index=False) |
| 332 |
|
| 333 |
def save_missing_psl_txt_results(self): |
| 334 |
""" |
| 335 |
Saves domains with invalid _psl TXT records to data/missing_psl_txt.csv. |
| 336 |
""" |
| 337 |
missing_psl_txt_df = self.df[self.df["psl_txt_status"] == "invalid"].sort_values(by="psl_entry") |
| 338 |
missing_psl_txt_df.to_csv("data/missing_psl_txt.csv", index=False) |
| 339 |
|
| 340 |
def save_expiry_less_than_2yrs_results(self): |
| 341 |
""" |
| 342 |
Saves domains with WHOIS expiry date less than 2 years from now to data/expiry_less_than_2yrs.csv. |
| 343 |
""" |
| 344 |
expiry_less_than_2yrs_df = self.df[self.df["expiry_check_status"] == "FAIL_2Y"].sort_values(by="psl_entry") |
| 345 |
expiry_less_than_2yrs_df.to_csv("data/expiry_less_than_2yrs.csv", index=False) |
| 346 |
|
| 347 |
def run(self): |
| 348 |
""" |
| 349 |
Executes the entire processing pipeline. |
| 350 |
""" |
| 351 |
psl_data = self.fetch_psl_data() |
| 352 |
raw_domains, domains = self.parse_psl_data(psl_data) |
| 353 |
self.process_domains(raw_domains, domains) |
| 354 |
self.save_results() |
| 355 |
self.save_invalid_results() |
| 356 |
self.save_hold_results() |
| 357 |
self.save_missing_psl_txt_results() |
| 358 |
self.save_expiry_less_than_2yrs_results() |
| 359 |
|
| 360 |
|
| 361 |
if __name__ == "__main__": |
| 362 |
processor = PSLPrivateDomainsProcessor() |
| 363 |
processor.run() |