retoor/devplacepy
1
0
Fork 1
Code Issues 44 Pull Requests 4 Actions Packages Projects Releases Wiki Activity

Update

Browse Source
This commit is contained in:
retoor
2026-07-09 02:52:54 +02:00
parent 818568c609
commit 48bb6c2ec2
95 changed files with 6115 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files
devplacepy/services/devii/fetch/controller.py
+37 -60
View File
@@ -21,32 +21,6 @@ from ..text import html_to_text
logger = logging.getLogger("devii.fetch")
CHROME_VERSION = "131"
CHROME_VERSION_FULL = "131.0.0.0"
STEALTH_HEADERS = {
"User-Agent": (
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
f"(KHTML, like Gecko) Chrome/{CHROME_VERSION_FULL} Safari/537.36"
),
"Accept": (
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,"
"image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"
),
"Accept-Language": "en-US,en;q=0.9",
"sec-ch-ua": (
f'"Google Chrome";v="{CHROME_VERSION}", "Chromium";v="{CHROME_VERSION}", '
'"Not_A Brand";v="24"'
),
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": '"Linux"',
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
"Upgrade-Insecure-Requests": "1",
"Cache-Control": "max-age=0",
"DNT": "1",
}
TITLE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
MIN_FETCH_CHARS = 1000
ALLOWED_METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS")
@@ -230,9 +204,7 @@ class FetchController:
raise_5xx: bool = False,
) -> tuple[str, str, str, int, dict[str, str]]:
limit = self._settings.fetch_max_bytes
merged_headers = dict(STEALTH_HEADERS)
if headers:
merged_headers.update({str(k): str(v) for k, v in headers.items()})
merged_headers = {str(k): str(v) for k, v in headers.items()} if headers else {}
request_kwargs: dict[str, Any] = {}
if json_body is not None:
request_kwargs["json"] = json_body
@@ -240,43 +212,48 @@ class FetchController:
request_kwargs["data"] = form
elif content is not None:
request_kwargs["content"] = content
async def _once(client: httpx.AsyncClient, target: str) -> tuple[str, str, str, int, dict[str, str]]:
async with client.stream(method, target, **request_kwargs) as response:
if raise_5xx and response.status_code >= 500:
raise UpstreamError(
f"Server returned {response.status_code}.",
status=response.status_code,
url=target,
)
chunks: list[bytes] = []
total = 0
async for chunk in response.aiter_bytes():
chunks.append(chunk)
total += len(chunk)
if total >= limit:
break
raw = b"".join(chunks)[:limit]
encoding = response.encoding or "utf-8"
try:
body = raw.decode(encoding, errors="replace")
except LookupError:
body = raw.decode("utf-8", errors="replace")
content_type = response.headers.get("content-type", "").lower()
response_headers = {key: value for key, value in response.headers.items()}
return body, str(response.url), content_type, response.status_code, response_headers
try:
async with stealth.stealth_async_client(
headers=merged_headers,
follow_redirects=True,
timeout=self._settings.fetch_timeout_seconds,
) as client:
async with client.stream(method, url, **request_kwargs) as response:
if raise_5xx and response.status_code >= 500:
raise UpstreamError(
f"Server returned {response.status_code}.",
status=response.status_code,
url=url,
)
chunks: list[bytes] = []
total = 0
async for chunk in response.aiter_bytes():
chunks.append(chunk)
total += len(chunk)
if total >= limit:
break
raw = b"".join(chunks)[:limit]
encoding = response.encoding or "utf-8"
try:
body = raw.decode(encoding, errors="replace")
except LookupError:
body = raw.decode("utf-8", errors="replace")
content_type = response.headers.get("content-type", "").lower()
response_headers = {
key: value for key, value in response.headers.items()
}
return (
body,
str(response.url),
content_type,
response.status_code,
response_headers,
)
result = await _once(client, url)
if method == "GET" and "html" in result[2]:
gate_url = stealth.detect_consent_gate(result[0])
if gate_url:
try:
await client.get(gate_url)
result = await _once(client, url)
except httpx.HTTPError as exc:
logger.debug("consent gate follow-up failed for %s: %s", url, exc)
return result
except httpx.TimeoutException as exc:
raise NetworkError(f"Request timed out fetching {url}", url=url) from exc
except httpx.HTTPError as exc: