Files
devplacepy/devplacepy/services/nodes/geo.py
T
2026-11-05 17:59:21 +01:00

298 lines
10 KiB
Python

# retoor <retoor@molodetz.nl>
from __future__ import annotations
import asyncio
import csv
import gzip
import ipaddress
import logging
import os
import sqlite3
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from devplacepy import config
from devplacepy.services.nodes.addressclass import classify
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class GeoSpec:
asn_url: str = "https://download.db-ip.com/free/dbip-asn-lite-{month}.csv.gz"
country_url: str = "https://download.db-ip.com/free/dbip-country-lite-{month}.csv.gz"
attribution: str = "IP geolocation by DB-IP.com (CC BY 4.0)"
max_rows: int = 2_000_000
min_asn_rows: int = 100_000
min_country_rows: int = 100_000
insert_chunk: int = 5000
download_timeout_seconds: int = 180
max_download_bytes: int = 64 * 1024 * 1024
months_back: int = 2
checks: tuple[tuple[str, int], ...] = (("1.1.1.1", 13335), ("8.8.8.8", 15169))
SPEC = GeoSpec()
SCHEMA = (
"CREATE TABLE geo_asn (family INTEGER NOT NULL, first BLOB NOT NULL, last BLOB NOT NULL, "
"asn INTEGER NOT NULL, org TEXT NOT NULL, PRIMARY KEY (family, first)) WITHOUT ROWID",
"CREATE TABLE geo_country (family INTEGER NOT NULL, first BLOB NOT NULL, last BLOB NOT NULL, "
"country TEXT NOT NULL, PRIMARY KEY (family, first)) WITHOUT ROWID",
"CREATE TABLE geo_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)",
)
UNKNOWN_COUNTRY = "ZZ"
class GeoError(RuntimeError):
pass
@dataclass(frozen=True)
class GeoResult:
country: str = ""
asn: int = 0
org: str = ""
asn_class: str = "unknown"
def pack(address: str) -> tuple[int, bytes]:
ip = ipaddress.ip_address(address.strip().strip("[]"))
if ip.version == 6 and ip.ipv4_mapped is not None:
ip = ip.ipv4_mapped
return ip.version, ip.packed
class GeoDatabase:
def __init__(self, path: Path | None = None) -> None:
self.path = Path(path) if path else config.NODE_GEO_DB
self.warned = False
def available(self) -> bool:
return self.path.is_file() and self.path.stat().st_size > 0
def _connect(self) -> sqlite3.Connection:
return sqlite3.connect(f"file:{self.path}?mode=ro", uri=True, timeout=5)
def _range(self, conn: sqlite3.Connection, table: str, columns: str, family: int, packed: bytes):
row = conn.execute(
f"SELECT {columns}, last FROM {table} WHERE family = ? AND first <= ? "
"ORDER BY first DESC LIMIT 1",
(family, packed),
).fetchone()
if row is None or row[-1] < packed:
return None
return row[:-1]
def lookup(self, address: str) -> GeoResult:
if not address or not self.available():
return GeoResult()
try:
family, packed = pack(address)
except ValueError:
return GeoResult()
try:
conn = self._connect()
except sqlite3.Error:
return GeoResult()
try:
asn_row = self._range(conn, "geo_asn", "asn, org", family, packed)
country_row = self._range(conn, "geo_country", "country", family, packed)
except sqlite3.Error:
return GeoResult()
finally:
conn.close()
asn, org = (int(asn_row[0]), str(asn_row[1])) if asn_row else (0, "")
country = str(country_row[0]) if country_row else ""
country = "" if country.upper() == UNKNOWN_COUNTRY else country
return GeoResult(country=country[:2].upper(), asn=asn, org=org, asn_class=classify(asn, org))
def status(self) -> dict:
if not self.available():
return {"available": False, "path": str(self.path)}
conn = self._connect()
try:
meta = dict(conn.execute("SELECT key, value FROM geo_meta").fetchall())
except sqlite3.Error:
meta = {}
finally:
conn.close()
return {"available": True, "path": str(self.path), **meta, "attribution": SPEC.attribution}
def months(now: datetime | None = None) -> list[str]:
moment = now or datetime.now(timezone.utc)
found = []
for back in range(SPEC.months_back + 1):
year, month = moment.year, moment.month - back
while month < 1:
month += 12
year -= 1
found.append(f"{year:04d}-{month:02d}")
return found
def parse_asn_rows(lines):
for row in csv.reader(lines):
if len(row) < 4:
continue
first, last, asn, org = row[0], row[1], row[2], row[3]
try:
family, low = pack(first)
_, high = pack(last)
number = int(asn)
except ValueError:
continue
yield family, low, high, number, org[:200]
def parse_country_rows(lines):
for row in csv.reader(lines):
if len(row) < 3:
continue
try:
family, low = pack(row[0])
_, high = pack(row[1])
except ValueError:
continue
yield family, low, high, row[2][:2].upper()
def _insert(conn: sqlite3.Connection, sql: str, rows) -> int:
count = 0
batch = []
for row in rows:
batch.append(row)
count += 1
if count > SPEC.max_rows:
raise GeoError(f"more than {SPEC.max_rows} rows, refusing to load")
if len(batch) >= SPEC.insert_chunk:
conn.executemany(sql, batch)
batch = []
if batch:
conn.executemany(sql, batch)
return count
def build(asn_lines, country_lines, target: Path, month: str) -> dict:
temporary = target.with_suffix(".tmp")
temporary.unlink(missing_ok=True)
conn = sqlite3.connect(temporary)
try:
for statement in SCHEMA:
conn.execute(statement)
asn_rows = _insert(conn, "INSERT OR REPLACE INTO geo_asn VALUES (?,?,?,?,?)", parse_asn_rows(asn_lines))
country_rows = _insert(conn, "INSERT OR REPLACE INTO geo_country VALUES (?,?,?,?)", parse_country_rows(country_lines))
if asn_rows < SPEC.min_asn_rows or country_rows < SPEC.min_country_rows:
raise GeoError(
f"dataset too small (asn {asn_rows}, country {country_rows}), refusing to replace the current one"
)
conn.executemany(
"INSERT INTO geo_meta VALUES (?,?)",
[
("month", month),
("loaded_at", datetime.now(timezone.utc).isoformat()),
("asn_rows", str(asn_rows)),
("country_rows", str(country_rows)),
],
)
conn.commit()
except Exception:
conn.close()
temporary.unlink(missing_ok=True)
raise
conn.close()
verify(temporary)
return {"month": month, "asn_rows": asn_rows, "country_rows": country_rows, "temporary": temporary}
def verify(path: Path) -> None:
probe = GeoDatabase(path)
for address, expected in SPEC.checks:
found = probe.lookup(address).asn
if found != expected:
path.unlink(missing_ok=True)
raise GeoError(f"sanity check failed: {address} resolved to AS{found}, expected AS{expected}")
def install(built: dict, target: Path) -> None:
os.replace(built["temporary"], target)
async def _download(url: str, destination: Path) -> bool:
from devplacepy.stealth import stealth_async_client
async with stealth_async_client(timeout=SPEC.download_timeout_seconds, follow_redirects=True) as client:
async with client.stream("GET", url) as response:
if response.status_code == 404:
return False
if response.status_code != 200:
raise GeoError(f"{url} answered {response.status_code}")
total = 0
handle = await asyncio.to_thread(open, destination, "wb")
try:
async for chunk in response.aiter_bytes():
total += len(chunk)
if total > SPEC.max_download_bytes:
raise GeoError(f"{url} exceeds {SPEC.max_download_bytes} bytes")
await asyncio.to_thread(handle.write, chunk)
finally:
await asyncio.to_thread(handle.close)
return True
async def fetch_month(month: str, workdir: Path) -> tuple[Path, Path] | None:
asn_file = workdir / f"asn-{month}.csv.gz"
country_file = workdir / f"country-{month}.csv.gz"
if not await _download(SPEC.asn_url.format(month=month), asn_file):
return None
if not await _download(SPEC.country_url.format(month=month), country_file):
return None
return asn_file, country_file
def build_from_files(asn_file: Path, country_file: Path, target: Path, month: str) -> dict:
with gzip.open(asn_file, "rt", encoding="utf-8", newline="") as asn_handle, gzip.open(
country_file, "rt", encoding="utf-8", newline=""
) as country_handle:
return build(asn_handle, country_handle, target, month)
async def geo_sync(*, dry_run: bool = False, month: str = "", target: Path | None = None, workdir: Path | None = None) -> dict:
destination = Path(target) if target else config.NODE_GEO_DB
destination.parent.mkdir(parents=True, exist_ok=True)
scratch = Path(workdir) if workdir else destination.parent
scratch.mkdir(parents=True, exist_ok=True)
candidates = [month] if month else months()
for candidate in candidates:
files = await fetch_month(candidate, scratch)
if files is None:
continue
try:
built = build_from_files(files[0], files[1], destination, candidate)
finally:
for item in files:
item.unlink(missing_ok=True)
summary = {key: value for key, value in built.items() if key != "temporary"}
if dry_run:
built["temporary"].unlink(missing_ok=True)
return {**summary, "installed": False}
install(built, destination)
return {**summary, "installed": True}
raise GeoError(f"no dataset found for {', '.join(candidates)}")
_shared: GeoDatabase | None = None
def shared_geo() -> GeoDatabase:
global _shared
if _shared is None or _shared.path != config.NODE_GEO_DB:
_shared = GeoDatabase()
return _shared