ticket #104 attempt 1

devplacepy/docs_api/_shared.py

@@ -38,7 +38,6 @@ def field(
     example="",
     description="",
     options=None,
-    nullable=False,
 ):
     spec = {
         "name": name,
@@ -47,7 +46,6 @@ def field(
         "required": required,
         "example": example,
         "description": description,
-        "nullable": nullable,
     }
     if options:
         spec["options"] = list(options)

devplacepy/docs_examples.py

@@ -39,15 +39,7 @@ def _unwrap_optional(annotation):
     return annotation
 
 
-def _is_optional(annotation):
-    if get_origin(annotation) is Union:
-        return type(None) in get_args(annotation)
-    return False
-
-
-def _value(name, annotation, stack, is_nullable=False):
-    if is_nullable:
-        return None
+def _value(name, annotation, stack):
     annotation = _unwrap_optional(annotation)
     origin = get_origin(annotation)
     if origin in (list, set, tuple):
@@ -79,8 +71,7 @@ def _from_model(model, stack):
     stack = stack | {model}
     example = {}
     for name, info in model.model_fields.items():
-        is_nullable = _is_optional(info.annotation)
-        example[name] = _value(name, info.annotation, stack, is_nullable)
+        example[name] = _value(name, info.annotation, stack)
     return example
 
 

devplacepy/services/jobs/deepsearch/crawl.py

@@ -22,6 +22,8 @@ from .pdf import MAX_PDF_BYTES, extract_pdf_text, is_pdf
 
 logger = logging.getLogger(__name__)
 
+_pw_lock = asyncio.Lock()
+
 RSEARCH_URL = "https://rsearch.app.molodetz.nl"
 RSEARCH_TIMEOUT_SECONDS = 45.0
 FETCH_TIMEOUT_SECONDS = 20.0
@@ -172,13 +174,7 @@ async def _render_with_playwright(
 ) -> tuple[str, str, int, list[tuple[str, str]]]:
     from playwright.async_api import async_playwright
 
-    own_browser = browser is None
-    if own_browser:
-        pw = await async_playwright().__aenter__()
-        browser = await pw.chromium.launch(
-            headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
-        )
-    try:
+    async def _render(browser) -> tuple[str, str, int, list[tuple[str, str]]]:
         context = await browser.new_context(user_agent=USER_AGENT)
         page = await context.new_page()
         response = await page.goto(url, wait_until="load", timeout=30000)
@@ -189,10 +185,18 @@ async def _render_with_playwright(
         await context.close()
         extracted = extract_html(content, base_url=url)
         return extracted.title, extracted.text, status, extracted.links
+
+    if browser is None:
+        async with async_playwright() as pw:
+            browser = await pw.chromium.launch(
+                headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
+            )
+            try:
+                return await _render(browser)
             finally:
-        if own_browser:
                 await browser.close()
-            await pw.__aexit__(None, None, None)
+    else:
+        return await _render(browser)
 
 
 async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
@@ -242,8 +246,12 @@ async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
         except (LookupError, ValueError) as exc:
             logger.info("deepsearch decode failed for %s: %s", url, exc)
     if len(text) < MIN_PAGE_CHARS:
+        async with _pw_lock:
             try:
                 r_title, r_text, r_status, r_links = await _render_with_playwright(url, browser)
+            except Exception as exc:
+                logger.info("deepsearch render failed for %s: %s", url, exc)
+                r_title, r_text, r_status, r_links = "", "", 0, []
         if len(r_text) > len(text):
             title, text, status, source, links = (
                 r_title or title,
@@ -252,8 +260,6 @@ async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
                 "playwright",
                 r_links,
             )
-        except Exception as exc:
-            logger.info("deepsearch render failed for %s: %s", url, exc)
     if len(text) < MIN_PAGE_CHARS:
         return None
     return CrawledPage(
@@ -296,10 +302,9 @@ async def crawl(
     total = min(len(level_candidates), max_pages)
     cancelled = False
 
-    pw = None
     browser = None
+    async with async_playwright() as pw:
         try:
-        pw = await async_playwright().__aenter__()
             browser = await pw.chromium.launch(
                 headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
             )
@@ -392,6 +397,4 @@ async def crawl(
         finally:
             if browser is not None:
                 await browser.close()
-        if pw is not None:
-            await pw.__aexit__(None, None, None)
     return outcome

devplacepy/static/css/docs.css

@@ -218,16 +218,6 @@
     font-weight: 700;
 }
 
-.param-nullable {
-    font-size: 0.625rem;
-    text-transform: uppercase;
-    letter-spacing: 0.05em;
-    color: var(--text-muted);
-    border: 1px dashed var(--border);
-    border-radius: 999px;
-    padding: 0.05rem 0.4rem;
-}
-
 .param-loc {
     font-size: 0.625rem;
     text-transform: uppercase;

devplacepy/static/js/ApiTester.js

@@ -139,7 +139,6 @@ export class ApiTester {
             const label = this.el("div", { class: "param-label" }, [
                 this.el("span", { class: "param-name", text: param.name }),
                 param.required ? this.el("span", { class: "param-required", text: "*" }) : null,
-                param.nullable ? this.el("span", { class: "param-nullable", text: "nullable" }) : null,
                 this.el("span", { class: "param-loc param-loc-" + param.location, text: param.location }),
             ]);
             const allowed = param.type === "enum" && param.options && param.options.length

tests/unit/test_docs_nullable.py

@@ -1,59 +0,0 @@
-# retoor <retoor@molodetz.nl>
-
-from typing import Optional
-
-from pydantic import BaseModel
-
-from devplacepy.docs_api._shared import field
-from devplacepy.docs_examples import _is_optional, schema_example
-
-
-def test_field_nullable_parameter():
-    f = field("bio", "query", "string", False, "", "User biography", nullable=True)
-    assert f["nullable"] is True
-
-    f2 = field("username", "path", "string", True, "alice", "Target username")
-    assert f2["nullable"] is False
-
-
-def test_is_optional_optional_type():
-    assert _is_optional(Optional[str]) is True
-    assert _is_optional(Optional[int]) is True
-    assert _is_optional(Optional[list[str]]) is True
-
-
-def test_is_optional_non_optional():
-    assert _is_optional(str) is False
-    assert _is_optional(int) is False
-    assert _is_optional(list[str]) is False
-    assert _is_optional(dict) is False
-
-
-def test_schema_example_nullable_fields():
-    class NullableModel(BaseModel):
-        name: str
-        bio: Optional[str] = None
-        age: int
-        score: Optional[int] = None
-        tags: Optional[list[str]] = None
-
-    result = schema_example(NullableModel)
-
-    assert result["name"] == "string"
-    assert result["age"] == 0
-    assert result["bio"] is None
-    assert result["score"] is None
-    assert result["tags"] is None
-
-
-def test_schema_example_non_nullable_fields_unaffected():
-    class StrictModel(BaseModel):
-        x: str
-        y: int
-        z: bool
-
-    result = schema_example(StrictModel)
-
-    assert result["x"] == "string"
-    assert result["y"] == 0
-    assert result["z"] is False