63 lines
2.4 KiB
Python
63 lines
2.4 KiB
Python
|
# retoor <retoor@molodetz.nl>
|
||
|
|
import re
|
||
|
|
from urllib.parse import unquote
|
||
|
|
|
||
|
|
import httpx
|
||
|
|
|
||
|
|
from molodetz import config
|
||
|
|
from molodetz.database import get_setting
|
||
|
|
from molodetz.curl_transport import CURL_AVAILABLE, CurlImpersonationTransport
|
||
|
|
from molodetz.net_guard import GuardedTransport
|
||
|
|
|
||
|
|
BROWSER_ALIGNED_HEADERS = {
|
||
|
|
"user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||
|
|
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
|
|
"accept-language": "nl-NL,nl;q=0.9,en;q=0.8",
|
||
|
|
}
|
||
|
|
|
||
|
|
CONSENT_MARKERS = ("consent", "privacy-gate", "cookiewall")
|
||
|
|
CONSENT_REDIRECT = re.compile(r"""(?:window\.)?location(?:\.href)?\s*=\s*decodeURIComponent\(\s*['"]([^'"]+)['"]""")
|
||
|
|
|
||
|
|
|
||
|
|
def configured_proxy():
|
||
|
|
try:
|
||
|
|
value = get_setting("outbound_proxy_url", "")
|
||
|
|
except Exception:
|
||
|
|
value = ""
|
||
|
|
return value or config.OUTBOUND_PROXY_URL or None
|
||
|
|
|
||
|
|
|
||
|
|
def _async_transport(proxy, timeout):
|
||
|
|
if CURL_AVAILABLE:
|
||
|
|
return CurlImpersonationTransport(proxy=proxy, timeout=timeout)
|
||
|
|
return httpx.AsyncHTTPTransport(http2=True, proxy=proxy, retries=1)
|
||
|
|
|
||
|
|
|
||
|
|
def stealth_async_client(proxy=None, timeout=30.0, **kwargs):
|
||
|
|
proxy = proxy if proxy is not None else configured_proxy()
|
||
|
|
transport = _async_transport(proxy, timeout)
|
||
|
|
headers = None if CURL_AVAILABLE else BROWSER_ALIGNED_HEADERS
|
||
|
|
return httpx.AsyncClient(transport=transport, timeout=timeout, headers=headers, follow_redirects=True, **kwargs)
|
||
|
|
|
||
|
|
|
||
|
|
def stealth_sync_client(proxy=None, timeout=30.0, **kwargs):
|
||
|
|
proxy = proxy if proxy is not None else configured_proxy()
|
||
|
|
return httpx.Client(http2=True, proxy=proxy, timeout=timeout, headers=BROWSER_ALIGNED_HEADERS, follow_redirects=True, **kwargs)
|
||
|
|
|
||
|
|
|
||
|
|
def guarded_async_client(proxy=None, timeout=20.0, **kwargs):
|
||
|
|
proxy = proxy if proxy is not None else configured_proxy()
|
||
|
|
transport = GuardedTransport(_async_transport(proxy, timeout))
|
||
|
|
headers = None if CURL_AVAILABLE else BROWSER_ALIGNED_HEADERS
|
||
|
|
return httpx.AsyncClient(transport=transport, timeout=timeout, headers=headers, follow_redirects=True, max_redirects=5, **kwargs)
|
||
|
|
|
||
|
|
|
||
|
|
def detect_consent_gate(html):
|
||
|
|
lowered = (html or "").lower()
|
||
|
|
if not any(marker in lowered for marker in CONSENT_MARKERS):
|
||
|
|
return None
|
||
|
|
match = CONSENT_REDIRECT.search(html)
|
||
|
|
if not match:
|
||
|
|
return None
|
||
|
|
return unquote(match.group(1))
|