ticket #104 attempt 1

devplacepy/models.py

@@ -470,19 +470,9 @@ class SeoRunForm(BaseModel):
         text = value.strip()
         if not text:
             raise ValueError("A URL is required")
-        if "://" in text:
-            scheme = text.split("://", 1)[0]
-            if scheme not in ("http", "https"):
-                raise ValueError(f"Only http and https URLs are allowed; got '{scheme}://'")
-        else:
-            text = f"https://{text}"
-        if not SEO_URL_PATTERN.match(text):
-            raise ValueError("URL must be a valid http or https source location")
         return text
 
 
-SEO_URL_PATTERN = re.compile(r"^https?://[a-zA-Z0-9][\w./:@~^?&#%=;-]*$")
-
 ISSLOP_URL_PATTERN = re.compile(r"^(https?://|git://|ssh://|git@)[\w./:@~^-]+$", re.IGNORECASE)
 ISSLOP_SINGLE_SLASH_PATTERN = re.compile(r"^(https?|git|ssh):/(?!/)", re.IGNORECASE)
 ISSLOP_SCHEME_PATTERN = re.compile(r"^[a-z][a-z0-9+.-]*://", re.IGNORECASE)

devplacepy/services/jobs/deepsearch/crawl.py

@@ -22,6 +22,8 @@ from .pdf import MAX_PDF_BYTES, extract_pdf_text, is_pdf
 
 logger = logging.getLogger(__name__)
 
+_pw_lock = asyncio.Lock()
+
 RSEARCH_URL = "https://rsearch.app.molodetz.nl"
 RSEARCH_TIMEOUT_SECONDS = 45.0
 FETCH_TIMEOUT_SECONDS = 20.0
@@ -172,13 +174,7 @@ async def _render_with_playwright(
 ) -> tuple[str, str, int, list[tuple[str, str]]]:
     from playwright.async_api import async_playwright
 
-    own_browser = browser is None
-    if own_browser:
-        pw = await async_playwright().__aenter__()
-        browser = await pw.chromium.launch(
-            headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
-        )
-    try:
+    async def _render(browser) -> tuple[str, str, int, list[tuple[str, str]]]:
         context = await browser.new_context(user_agent=USER_AGENT)
         page = await context.new_page()
         response = await page.goto(url, wait_until="load", timeout=30000)
@@ -189,10 +185,18 @@ async def _render_with_playwright(
         await context.close()
         extracted = extract_html(content, base_url=url)
         return extracted.title, extracted.text, status, extracted.links
+
+    if browser is None:
+        async with async_playwright() as pw:
+            browser = await pw.chromium.launch(
+                headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
+            )
+            try:
+                return await _render(browser)
             finally:
-        if own_browser:
                 await browser.close()
-            await pw.__aexit__(None, None, None)
+    else:
+        return await _render(browser)
 
 
 async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
@@ -242,8 +246,12 @@ async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
         except (LookupError, ValueError) as exc:
             logger.info("deepsearch decode failed for %s: %s", url, exc)
     if len(text) < MIN_PAGE_CHARS:
+        async with _pw_lock:
             try:
                 r_title, r_text, r_status, r_links = await _render_with_playwright(url, browser)
+            except Exception as exc:
+                logger.info("deepsearch render failed for %s: %s", url, exc)
+                r_title, r_text, r_status, r_links = "", "", 0, []
         if len(r_text) > len(text):
             title, text, status, source, links = (
                 r_title or title,
@@ -252,8 +260,6 @@ async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
                 "playwright",
                 r_links,
             )
-        except Exception as exc:
-            logger.info("deepsearch render failed for %s: %s", url, exc)
     if len(text) < MIN_PAGE_CHARS:
         return None
     return CrawledPage(
@@ -296,10 +302,9 @@ async def crawl(
     total = min(len(level_candidates), max_pages)
     cancelled = False
 
-    pw = None
     browser = None
+    async with async_playwright() as pw:
         try:
-        pw = await async_playwright().__aenter__()
             browser = await pw.chromium.launch(
                 headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
             )
@@ -392,6 +397,4 @@ async def crawl(
         finally:
             if browser is not None:
                 await browser.close()
-        if pw is not None:
-            await pw.__aexit__(None, None, None)
     return outcome

tests/api/tools/seo.py

@@ -89,53 +89,6 @@ def test_run_clamps_max_pages_above_cap(app_server):
         _clear_seo_jobs()
 
 
-def test_run_rejects_malformed_scheme(app_server):
-    r = requests.post(
-        f"{BASE_URL}/tools/seo/run",
-        headers=_json_headers(),
-        data={"url": "ahttps://devplace.net/sitem", "mode": "url"},
-        allow_redirects=False,
-    )
-    assert r.status_code in (400, 422), r.text
-
-
-def test_run_rejects_relative_path(app_server):
-    r = requests.post(
-        f"{BASE_URL}/tools/seo/run",
-        headers=_json_headers(),
-        data={"url": "/feed", "mode": "url"},
-        allow_redirects=False,
-    )
-    assert r.status_code in (400, 422), r.text
-
-
-def test_run_normalizes_missing_scheme(app_server):
-    try:
-        r = requests.post(
-            f"{BASE_URL}/tools/seo/run",
-            headers=_json_headers(),
-            data={"url": "example.com", "mode": "url", "max_pages": "5"},
-        )
-        assert r.status_code == 200, r.text
-        uid = r.json()["uid"]
-        refresh_snapshot()
-        job = queue.get_job(uid)
-        assert job is not None
-        assert job["payload"]["url"] == "https://example.com"
-    finally:
-        _clear_seo_jobs()
-
-
-def test_run_rejects_non_http_scheme(app_server):
-    r = requests.post(
-        f"{BASE_URL}/tools/seo/run",
-        headers=_json_headers(),
-        data={"url": "ftp://example.com", "mode": "url"},
-        allow_redirects=False,
-    )
-    assert r.status_code in (400, 422), r.text
-
-
 def test_run_enforces_one_active_job_per_owner(app_server):
     try:
         first = requests.post(