DevPlace CI / test (push) Failing after 1h5m29s
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
298 lines
10 KiB
Python
298 lines
10 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import csv
|
|
import gzip
|
|
import ipaddress
|
|
import logging
|
|
import os
|
|
import sqlite3
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from devplacepy import config
|
|
from devplacepy.services.nodes.addressclass import classify
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class GeoSpec:
|
|
asn_url: str = "https://download.db-ip.com/free/dbip-asn-lite-{month}.csv.gz"
|
|
country_url: str = "https://download.db-ip.com/free/dbip-country-lite-{month}.csv.gz"
|
|
attribution: str = "IP geolocation by DB-IP.com (CC BY 4.0)"
|
|
max_rows: int = 2_000_000
|
|
min_asn_rows: int = 100_000
|
|
min_country_rows: int = 100_000
|
|
insert_chunk: int = 5000
|
|
download_timeout_seconds: int = 180
|
|
max_download_bytes: int = 64 * 1024 * 1024
|
|
months_back: int = 2
|
|
checks: tuple[tuple[str, int], ...] = (("1.1.1.1", 13335), ("8.8.8.8", 15169))
|
|
|
|
|
|
SPEC = GeoSpec()
|
|
|
|
SCHEMA = (
|
|
"CREATE TABLE geo_asn (family INTEGER NOT NULL, first BLOB NOT NULL, last BLOB NOT NULL, "
|
|
"asn INTEGER NOT NULL, org TEXT NOT NULL, PRIMARY KEY (family, first)) WITHOUT ROWID",
|
|
"CREATE TABLE geo_country (family INTEGER NOT NULL, first BLOB NOT NULL, last BLOB NOT NULL, "
|
|
"country TEXT NOT NULL, PRIMARY KEY (family, first)) WITHOUT ROWID",
|
|
"CREATE TABLE geo_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)",
|
|
)
|
|
|
|
|
|
UNKNOWN_COUNTRY = "ZZ"
|
|
|
|
|
|
class GeoError(RuntimeError):
|
|
pass
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class GeoResult:
|
|
country: str = ""
|
|
asn: int = 0
|
|
org: str = ""
|
|
asn_class: str = "unknown"
|
|
|
|
|
|
def pack(address: str) -> tuple[int, bytes]:
|
|
ip = ipaddress.ip_address(address.strip().strip("[]"))
|
|
if ip.version == 6 and ip.ipv4_mapped is not None:
|
|
ip = ip.ipv4_mapped
|
|
return ip.version, ip.packed
|
|
|
|
|
|
class GeoDatabase:
|
|
def __init__(self, path: Path | None = None) -> None:
|
|
self.path = Path(path) if path else config.NODE_GEO_DB
|
|
self.warned = False
|
|
|
|
def available(self) -> bool:
|
|
return self.path.is_file() and self.path.stat().st_size > 0
|
|
|
|
def _connect(self) -> sqlite3.Connection:
|
|
return sqlite3.connect(f"file:{self.path}?mode=ro", uri=True, timeout=5)
|
|
|
|
def _range(self, conn: sqlite3.Connection, table: str, columns: str, family: int, packed: bytes):
|
|
row = conn.execute(
|
|
f"SELECT {columns}, last FROM {table} WHERE family = ? AND first <= ? "
|
|
"ORDER BY first DESC LIMIT 1",
|
|
(family, packed),
|
|
).fetchone()
|
|
if row is None or row[-1] < packed:
|
|
return None
|
|
return row[:-1]
|
|
|
|
def lookup(self, address: str) -> GeoResult:
|
|
if not address or not self.available():
|
|
return GeoResult()
|
|
try:
|
|
family, packed = pack(address)
|
|
except ValueError:
|
|
return GeoResult()
|
|
try:
|
|
conn = self._connect()
|
|
except sqlite3.Error:
|
|
return GeoResult()
|
|
try:
|
|
asn_row = self._range(conn, "geo_asn", "asn, org", family, packed)
|
|
country_row = self._range(conn, "geo_country", "country", family, packed)
|
|
except sqlite3.Error:
|
|
return GeoResult()
|
|
finally:
|
|
conn.close()
|
|
asn, org = (int(asn_row[0]), str(asn_row[1])) if asn_row else (0, "")
|
|
country = str(country_row[0]) if country_row else ""
|
|
country = "" if country.upper() == UNKNOWN_COUNTRY else country
|
|
return GeoResult(country=country[:2].upper(), asn=asn, org=org, asn_class=classify(asn, org))
|
|
|
|
def status(self) -> dict:
|
|
if not self.available():
|
|
return {"available": False, "path": str(self.path)}
|
|
conn = self._connect()
|
|
try:
|
|
meta = dict(conn.execute("SELECT key, value FROM geo_meta").fetchall())
|
|
except sqlite3.Error:
|
|
meta = {}
|
|
finally:
|
|
conn.close()
|
|
return {"available": True, "path": str(self.path), **meta, "attribution": SPEC.attribution}
|
|
|
|
|
|
def months(now: datetime | None = None) -> list[str]:
|
|
moment = now or datetime.now(timezone.utc)
|
|
found = []
|
|
for back in range(SPEC.months_back + 1):
|
|
year, month = moment.year, moment.month - back
|
|
while month < 1:
|
|
month += 12
|
|
year -= 1
|
|
found.append(f"{year:04d}-{month:02d}")
|
|
return found
|
|
|
|
|
|
def parse_asn_rows(lines):
|
|
for row in csv.reader(lines):
|
|
if len(row) < 4:
|
|
continue
|
|
first, last, asn, org = row[0], row[1], row[2], row[3]
|
|
try:
|
|
family, low = pack(first)
|
|
_, high = pack(last)
|
|
number = int(asn)
|
|
except ValueError:
|
|
continue
|
|
yield family, low, high, number, org[:200]
|
|
|
|
|
|
def parse_country_rows(lines):
|
|
for row in csv.reader(lines):
|
|
if len(row) < 3:
|
|
continue
|
|
try:
|
|
family, low = pack(row[0])
|
|
_, high = pack(row[1])
|
|
except ValueError:
|
|
continue
|
|
yield family, low, high, row[2][:2].upper()
|
|
|
|
|
|
def _insert(conn: sqlite3.Connection, sql: str, rows) -> int:
|
|
count = 0
|
|
batch = []
|
|
for row in rows:
|
|
batch.append(row)
|
|
count += 1
|
|
if count > SPEC.max_rows:
|
|
raise GeoError(f"more than {SPEC.max_rows} rows, refusing to load")
|
|
if len(batch) >= SPEC.insert_chunk:
|
|
conn.executemany(sql, batch)
|
|
batch = []
|
|
if batch:
|
|
conn.executemany(sql, batch)
|
|
return count
|
|
|
|
|
|
def build(asn_lines, country_lines, target: Path, month: str) -> dict:
|
|
temporary = target.with_suffix(".tmp")
|
|
temporary.unlink(missing_ok=True)
|
|
conn = sqlite3.connect(temporary)
|
|
try:
|
|
for statement in SCHEMA:
|
|
conn.execute(statement)
|
|
asn_rows = _insert(conn, "INSERT OR REPLACE INTO geo_asn VALUES (?,?,?,?,?)", parse_asn_rows(asn_lines))
|
|
country_rows = _insert(conn, "INSERT OR REPLACE INTO geo_country VALUES (?,?,?,?)", parse_country_rows(country_lines))
|
|
if asn_rows < SPEC.min_asn_rows or country_rows < SPEC.min_country_rows:
|
|
raise GeoError(
|
|
f"dataset too small (asn {asn_rows}, country {country_rows}), refusing to replace the current one"
|
|
)
|
|
conn.executemany(
|
|
"INSERT INTO geo_meta VALUES (?,?)",
|
|
[
|
|
("month", month),
|
|
("loaded_at", datetime.now(timezone.utc).isoformat()),
|
|
("asn_rows", str(asn_rows)),
|
|
("country_rows", str(country_rows)),
|
|
],
|
|
)
|
|
conn.commit()
|
|
except Exception:
|
|
conn.close()
|
|
temporary.unlink(missing_ok=True)
|
|
raise
|
|
conn.close()
|
|
verify(temporary)
|
|
return {"month": month, "asn_rows": asn_rows, "country_rows": country_rows, "temporary": temporary}
|
|
|
|
|
|
def verify(path: Path) -> None:
|
|
probe = GeoDatabase(path)
|
|
for address, expected in SPEC.checks:
|
|
found = probe.lookup(address).asn
|
|
if found != expected:
|
|
path.unlink(missing_ok=True)
|
|
raise GeoError(f"sanity check failed: {address} resolved to AS{found}, expected AS{expected}")
|
|
|
|
|
|
def install(built: dict, target: Path) -> None:
|
|
os.replace(built["temporary"], target)
|
|
|
|
|
|
async def _download(url: str, destination: Path) -> bool:
|
|
from devplacepy.stealth import stealth_async_client
|
|
|
|
async with stealth_async_client(timeout=SPEC.download_timeout_seconds, follow_redirects=True) as client:
|
|
async with client.stream("GET", url) as response:
|
|
if response.status_code == 404:
|
|
return False
|
|
if response.status_code != 200:
|
|
raise GeoError(f"{url} answered {response.status_code}")
|
|
total = 0
|
|
handle = await asyncio.to_thread(open, destination, "wb")
|
|
try:
|
|
async for chunk in response.aiter_bytes():
|
|
total += len(chunk)
|
|
if total > SPEC.max_download_bytes:
|
|
raise GeoError(f"{url} exceeds {SPEC.max_download_bytes} bytes")
|
|
await asyncio.to_thread(handle.write, chunk)
|
|
finally:
|
|
await asyncio.to_thread(handle.close)
|
|
return True
|
|
|
|
|
|
async def fetch_month(month: str, workdir: Path) -> tuple[Path, Path] | None:
|
|
asn_file = workdir / f"asn-{month}.csv.gz"
|
|
country_file = workdir / f"country-{month}.csv.gz"
|
|
if not await _download(SPEC.asn_url.format(month=month), asn_file):
|
|
return None
|
|
if not await _download(SPEC.country_url.format(month=month), country_file):
|
|
return None
|
|
return asn_file, country_file
|
|
|
|
|
|
def build_from_files(asn_file: Path, country_file: Path, target: Path, month: str) -> dict:
|
|
with gzip.open(asn_file, "rt", encoding="utf-8", newline="") as asn_handle, gzip.open(
|
|
country_file, "rt", encoding="utf-8", newline=""
|
|
) as country_handle:
|
|
return build(asn_handle, country_handle, target, month)
|
|
|
|
|
|
async def geo_sync(*, dry_run: bool = False, month: str = "", target: Path | None = None, workdir: Path | None = None) -> dict:
|
|
destination = Path(target) if target else config.NODE_GEO_DB
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
scratch = Path(workdir) if workdir else destination.parent
|
|
scratch.mkdir(parents=True, exist_ok=True)
|
|
candidates = [month] if month else months()
|
|
for candidate in candidates:
|
|
files = await fetch_month(candidate, scratch)
|
|
if files is None:
|
|
continue
|
|
try:
|
|
built = build_from_files(files[0], files[1], destination, candidate)
|
|
finally:
|
|
for item in files:
|
|
item.unlink(missing_ok=True)
|
|
summary = {key: value for key, value in built.items() if key != "temporary"}
|
|
if dry_run:
|
|
built["temporary"].unlink(missing_ok=True)
|
|
return {**summary, "installed": False}
|
|
install(built, destination)
|
|
return {**summary, "installed": True}
|
|
raise GeoError(f"no dataset found for {', '.join(candidates)}")
|
|
|
|
|
|
_shared: GeoDatabase | None = None
|
|
|
|
|
|
def shared_geo() -> GeoDatabase:
|
|
global _shared
|
|
if _shared is None or _shared.path != config.NODE_GEO_DB:
|
|
_shared = GeoDatabase()
|
|
return _shared
|
|
|