# retoor import re from urllib.parse import unquote import httpx from molodetz import config from molodetz.database import get_setting from molodetz.curl_transport import CURL_AVAILABLE, CurlImpersonationTransport from molodetz.net_guard import GuardedTransport BROWSER_ALIGNED_HEADERS = { "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "accept-language": "en-GB,en;q=0.9", } CONSENT_MARKERS = ("consent", "privacy-gate", "cookiewall") CONSENT_REDIRECT = re.compile(r"""(?:window\.)?location(?:\.href)?\s*=\s*decodeURIComponent\(\s*['"]([^'"]+)['"]""") def configured_proxy(): try: value = get_setting("outbound_proxy_url", "") except Exception: value = "" return value or config.OUTBOUND_PROXY_URL or None def _async_transport(proxy, timeout): if CURL_AVAILABLE: return CurlImpersonationTransport(proxy=proxy, timeout=timeout) return httpx.AsyncHTTPTransport(http2=True, proxy=proxy, retries=1) def stealth_async_client(proxy=None, timeout=30.0, **kwargs): proxy = proxy if proxy is not None else configured_proxy() transport = _async_transport(proxy, timeout) headers = None if CURL_AVAILABLE else BROWSER_ALIGNED_HEADERS return httpx.AsyncClient(transport=transport, timeout=timeout, headers=headers, follow_redirects=True, **kwargs) def stealth_sync_client(proxy=None, timeout=30.0, **kwargs): proxy = proxy if proxy is not None else configured_proxy() return httpx.Client(http2=True, proxy=proxy, timeout=timeout, headers=BROWSER_ALIGNED_HEADERS, follow_redirects=True, **kwargs) def guarded_async_client(proxy=None, timeout=20.0, **kwargs): proxy = proxy if proxy is not None else configured_proxy() transport = GuardedTransport(_async_transport(proxy, timeout)) headers = None if CURL_AVAILABLE else BROWSER_ALIGNED_HEADERS return httpx.AsyncClient(transport=transport, timeout=timeout, headers=headers, follow_redirects=True, max_redirects=5, **kwargs) def detect_consent_gate(html): lowered = (html or "").lower() if not any(marker in lowered for marker in CONSENT_MARKERS): return None match = CONSENT_REDIRECT.search(html) if not match: return None return unquote(match.group(1))