forked from retoor/devplacepy
feat: add seo diagnostics tool with cli commands, config paths, and static versioning
Add SEO_REPORTS_DIR to config, STATIC_VERSION for cache-busting, seo prune/clear CLI subcommands, tools router with seo job endpoints, and boot-versioned static URLs in Dockerfile and Makefile
This commit is contained in:
@@ -13,6 +13,7 @@ from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from devplacepy.net_guard import effective_address, is_blocked_address
|
||||
from ..config import Settings
|
||||
from ..errors import NetworkError, ToolInputError, UpstreamError
|
||||
from ..text import html_to_text
|
||||
@@ -49,21 +50,6 @@ TITLE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
|
||||
MIN_FETCH_CHARS = 1000
|
||||
ALLOWED_METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS")
|
||||
|
||||
NAT64_PREFIXES = (
|
||||
ipaddress.ip_network("64:ff9b::/96"),
|
||||
ipaddress.ip_network("64:ff9b:1::/48"),
|
||||
)
|
||||
|
||||
|
||||
def _effective_address(address: Any) -> Any:
|
||||
if isinstance(address, ipaddress.IPv6Address):
|
||||
if address.ipv4_mapped is not None:
|
||||
return address.ipv4_mapped
|
||||
for prefix in NAT64_PREFIXES:
|
||||
if address in prefix:
|
||||
return ipaddress.IPv4Address(int(address) & 0xFFFFFFFF)
|
||||
return address
|
||||
|
||||
|
||||
class FetchController:
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
@@ -218,17 +204,10 @@ class FetchController:
|
||||
except socket.gaierror as exc:
|
||||
raise NetworkError(f"Could not resolve host: {host}", host=host) from exc
|
||||
for info in infos:
|
||||
address = _effective_address(ipaddress.ip_address(info[4][0]))
|
||||
if (
|
||||
address.is_private
|
||||
or address.is_loopback
|
||||
or address.is_link_local
|
||||
or address.is_reserved
|
||||
or address.is_multicast
|
||||
or address.is_unspecified
|
||||
):
|
||||
address = ipaddress.ip_address(info[4][0])
|
||||
if is_blocked_address(address):
|
||||
raise ToolInputError(
|
||||
f"Refusing to fetch a private or local address ({address}). "
|
||||
f"Refusing to fetch a private or local address ({effective_address(address)}). "
|
||||
"Set DEVII_FETCH_ALLOW_PRIVATE=1 to override."
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user