from warcio.archiveiterator import ArchiveIterator
from urllib.parse import urlparse
import gzip
import json

input_file = "CC-MAIN-20251005114239-20251005144239-00000.warc.wat.gz"
output_file = "domains.txt"

domains = set()

with gzip.open(input_file, "rb") as stream:
    for record in ArchiveIterator(stream):
        if record.rec_type != "metadata":
            continue
        try:
            wat = json.loads(record.content_stream().read())
            url = wat["Envelope"]["WARC-Header-Metadata"].get("WARC-Target-URI")
            if url:
                host = urlparse(url).hostname
                if host:
                    domains.add(host.lower())
        except Exception:
            continue

# сохранить в txt по одному домену на строку
with open(output_file, "w", encoding="utf-8") as f:
    for d in sorted(domains):
        f.write(d + "\n")

print(f"✅ Сохранено {len(domains)} уникальных доменов в {output_file}")
