Add community node protocol spec, Agent Skills, and make the failing tests hermetic

Community nodes (Phase 1 of plan.md): the protocol specification in services/nodes/spec.py as the single source of truth, GET /nodes/spec.json, a generated protocol docs page, the Community nodes API docs group, and the implementation plan plus the residential proxy research. The no-open-proxy rule is enforced in the spec (open_proxy cannot be set). Agent Skills: register the Skills API docs group, cascade skills on account deletion, add the nested CLAUDE.md, README and routes docs, and cover the parser, the fetcher (end to end against a mock origin), the data layer and the routes with 73 unit and 16 api tests. Tests made hermetic with the repo's existing patterns instead of ambient environment: embeddings tests set OPENROUTER_API_KEY via monkeypatch, the workspace tests install FakeBackend via runtime.set_backend, and the skills install tests use run_async. DockerCliBackend now returns 127 (command not found) when the CLI is missing instead of raising FileNotFoundError, so a host without docker answers with a clean error rather than a 500. The Providers tab empty state no longer names the Model routes tab. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-14 07:13:25 +02:00
parent cae139ead9
commit f55c7ee5f4
102 changed files with 7414 additions and 65 deletions
@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>
@@ -0,0 +1,94 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.services.skills import limits
+from devplacepy.services.skills.fetcher import (
+    SkillFetchError,
+    classify_candidate,
+    decode_text,
+    directory_url,
+    extract_candidates,
+    normalize_input_url,
+    origin_of,
+)
+from devplacepy.services.skills.parser import SkillFormatError
+
+
+def test_normalize_adds_scheme_and_strips_fragment():
+    assert normalize_input_url("example.org/skill#frag") == "https://example.org/skill"
+    assert normalize_input_url("  http://example.org/a/SKILL.md ") == "http://example.org/a/SKILL.md"
+
+
+@pytest.mark.parametrize(
+    "raw",
+    ["", "   ", "ftp://example.org/x", "https://user:pw@example.org/x", "x" * (limits.URL_MAX_CHARS + 1)],
+)
+def test_normalize_rejects_bad_urls(raw):
+    with pytest.raises(SkillFetchError):
+        normalize_input_url(raw)
+
+
+@pytest.mark.parametrize(
+    "url,expected",
+    [
+        ("https://e.org/a/SKILL.md", "https://e.org/a/"),
+        ("https://e.org/a/index.html", "https://e.org/a/"),
+        ("https://e.org/a", "https://e.org/a/"),
+        ("https://e.org/a/", "https://e.org/a/"),
+        ("https://e.org", "https://e.org/"),
+    ],
+)
+def test_directory_url(url, expected):
+    assert directory_url(url) == expected
+
+
+def test_origin_uses_default_ports():
+    assert origin_of("https://Example.org/x") == ("https", "example.org", 443)
+    assert origin_of("http://example.org:8080/x") == ("http", "example.org", 8080)
+
+
+def test_candidates_come_from_links_and_code_spans_once():
+    body = (
+        "See [the script](scripts/run.py) and `references/guide.md`.\n"
+        "Again [dup](scripts/run.py#anchor) and [other](docs/readme.md) and `not a path`."
+    )
+    assert extract_candidates(body) == ["scripts/run.py", "references/guide.md"]
+
+
+@pytest.mark.parametrize(
+    "candidate,accepted",
+    [
+        ("scripts/run.py", True),
+        ("./references/guide.md", True),
+        ("assets/templates/SPEC.md", True),
+        ("assets/img/hero.png", False),
+        ("../scripts/run.py", False),
+        ("scripts/../../etc/passwd", False),
+        ("scripts/%2e%2e/x.py", False),
+        ("scripts/a.py?x=1", False),
+        ("/scripts/run.py", False),
+        ("https://evil.org/scripts/run.py", False),
+        ("scripts\\run.py", False),
+        ("other/run.py", False),
+        ("scripts", False),
+        ("scripts/sp ace.py", False),
+    ],
+)
+def test_classify_candidate_only_accepts_safe_relative_text_paths(candidate, accepted):
+    path, reason = classify_candidate(candidate)
+    assert (path is not None) is accepted
+    assert (reason == "") is accepted
+
+
+def test_overlong_path_is_rejected():
+    path, reason = classify_candidate("scripts/" + "a" * limits.BUNDLED_PATH_MAX_CHARS + ".py")
+    assert path is None and reason
+
+
+def test_decode_text():
+    assert decode_text("héllo".encode("utf-8")) == "héllo"
+    with pytest.raises(SkillFormatError):
+        decode_text(b"a\x00b")
+    with pytest.raises(SkillFormatError):
+        decode_text(b"\xff\xfe\xfa")
@@ -0,0 +1,179 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+
+import httpx
+import pytest
+
+from devplacepy.net_guard import BlockedAddressError
+from devplacepy.services.skills import fetcher, limits
+from devplacepy.services.skills.fetcher import SkillFetchError, fetch_skill
+from tests.conftest import run_async as run
+
+SKILL = """---
+name: demo-skill
+description: Demonstrates an install.
+license: MIT
+---
+# Demo
+
+Run `scripts/run.py`, read [the guide](references/guide.md), see `./assets/logo.png`,
+[outside](https://other.example/scripts/x.py), [missing](references/missing.md)
+and [escape](scripts/../../etc/passwd).
+"""
+
+
+def serve(routes):
+    seen = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen.append(str(request.url))
+        entry = routes.get(request.url.path)
+        if entry is None:
+            return httpx.Response(404, text="missing")
+        if callable(entry):
+            return entry(request)
+        status, body, content_type = entry
+        return httpx.Response(status, content=body, headers={"content-type": content_type})
+
+    return httpx.MockTransport(handler), seen
+
+
+@pytest.fixture
+def mock_origin(monkeypatch):
+    def install(routes):
+        transport, seen = serve(routes)
+
+        def client_factory(**kwargs):
+            return httpx.AsyncClient(
+                transport=transport,
+                follow_redirects=kwargs.get("follow_redirects", True),
+                max_redirects=kwargs.get("max_redirects", 20),
+                headers=kwargs.get("headers"),
+            )
+
+        monkeypatch.setattr(fetcher, "guarded_async_client", client_factory)
+        return seen
+
+    return install
+
+
+def text(body, content_type="text/markdown"):
+    return (200, body.encode("utf-8"), content_type)
+
+
+def test_direct_skill_url_installs_with_bundled_files_and_reasons(mock_origin):
+    seen = mock_origin(
+        {
+            "/pack/SKILL.md": text(SKILL),
+            "/pack/scripts/run.py": text("print('hi')", "text/x-python"),
+            "/pack/references/guide.md": text("# Guide"),
+        }
+    )
+    result = run(fetch_skill("https://skills.example/pack/SKILL.md"))
+    assert result.parsed.name == "demo-skill"
+    assert result.parsed.license == "MIT"
+    assert result.source_url == "https://skills.example/pack/SKILL.md"
+    assert result.files == {"scripts/run.py": "print('hi')", "references/guide.md": "# Guide"}
+    reasons = {item.path: item.reason for item in result.skipped}
+    assert "binary" in reasons["./assets/logo.png"]
+    assert "references/missing.md" in reasons and "404" in reasons["references/missing.md"]
+    assert not any("other.example" in url for url in seen)
+    assert not any("etc/passwd" in url for url in seen)
+
+
+def test_directory_url_resolves_to_skill_md(mock_origin):
+    mock_origin(
+        {
+            "/pack": (200, b"<html>index</html>", "text/html"),
+            "/pack/SKILL.md": text(SKILL),
+        }
+    )
+    result = run(fetch_skill("https://skills.example/pack"))
+    assert result.source_url == "https://skills.example/pack/SKILL.md"
+    assert result.parsed.name == "demo-skill"
+
+
+def test_index_page_redirect_resolves_to_its_directory(mock_origin):
+    def redirect(request):
+        return httpx.Response(302, headers={"location": "/pack/index.html"})
+
+    mock_origin(
+        {
+            "/start": redirect,
+            "/pack/index.html": (200, b"<html></html>", "text/html"),
+            "/pack/SKILL.md": text(SKILL),
+        }
+    )
+    result = run(fetch_skill("https://skills.example/start"))
+    assert result.source_url == "https://skills.example/pack/SKILL.md"
+
+
+def test_html_where_skill_md_was_expected_is_an_error(mock_origin):
+    mock_origin({"/SKILL.md": (200, b"<html>login</html>", "text/html; charset=utf-8")})
+    with pytest.raises(SkillFetchError, match="HTML page"):
+        run(fetch_skill("https://skills.example/SKILL.md"))
+
+
+def test_http_error_and_invalid_skill_are_reported(mock_origin):
+    mock_origin({"/gone/SKILL.md": (500, b"boom", "text/plain"), "/bad/SKILL.md": text("no frontmatter")})
+    with pytest.raises(SkillFetchError, match="500"):
+        run(fetch_skill("https://skills.example/gone/SKILL.md"))
+    with pytest.raises(SkillFetchError, match="frontmatter"):
+        run(fetch_skill("https://skills.example/bad/SKILL.md"))
+
+
+def test_oversize_skill_md_is_refused(mock_origin):
+    mock_origin({"/big/SKILL.md": text("x" * (limits.SKILL_MD_MAX_BYTES + 10))})
+    with pytest.raises(SkillFetchError, match="exceeds"):
+        run(fetch_skill("https://skills.example/big/SKILL.md"))
+
+
+def test_bundled_file_count_and_size_limits(mock_origin):
+    links = " ".join(f"`scripts/f{index}.py`" for index in range(limits.BUNDLED_MAX_FILES + 3))
+    routes = {"/many/SKILL.md": text(f"---\nname: many\ndescription: d\n---\n{links}")}
+    for index in range(limits.BUNDLED_MAX_FILES + 3):
+        routes[f"/many/scripts/f{index}.py"] = text("x = 1")
+    mock_origin(routes)
+    result = run(fetch_skill("https://skills.example/many/SKILL.md"))
+    assert len(result.files) == limits.BUNDLED_MAX_FILES
+    assert sum(1 for item in result.skipped if "count limit" in item.reason) == 3
+
+    big = "y" * limits.BUNDLED_MAX_FILE_BYTES
+    routes = {"/size/SKILL.md": text("---\nname: size\ndescription: d\n---\n`scripts/a.py` `scripts/b.py` `scripts/c.py` `scripts/d.py` `scripts/e.py`")}
+    for letter in "abcde":
+        routes[f"/size/scripts/{letter}.py"] = text(big)
+    mock_origin(routes)
+    result = run(fetch_skill("https://skills.example/size/SKILL.md"))
+    assert sum(len(content) for content in result.files.values()) <= limits.BUNDLED_MAX_TOTAL_BYTES
+    assert any("total size limit" in item.reason for item in result.skipped)
+
+
+def test_a_binary_bundled_file_with_a_text_extension_is_skipped(mock_origin):
+    mock_origin(
+        {
+            "/bin/SKILL.md": text("---\nname: bin\ndescription: d\n---\n`scripts/blob.py`"),
+            "/bin/scripts/blob.py": (200, b"abc\x00def", "text/x-python"),
+        }
+    )
+    result = run(fetch_skill("https://skills.example/bin/SKILL.md"))
+    assert result.files == {}
+    assert result.skipped[0].path == "scripts/blob.py"
+
+
+def test_a_blocked_address_is_a_fetch_error(monkeypatch):
+    def refuse(**kwargs):
+        raise BlockedAddressError("Refusing to reach a private address.")
+
+    monkeypatch.setattr(fetcher, "guarded_async_client", refuse)
+    with pytest.raises(SkillFetchError, match="private address"):
+        run(fetch_skill("http://10.0.0.1/SKILL.md"))
+
+
+def test_the_overall_deadline_is_enforced(monkeypatch):
+    async def slow_fetch(url):
+        await asyncio.sleep(5)
+
+    monkeypatch.setattr(fetcher, "_fetch", slow_fetch)
+    with pytest.raises(SkillFetchError, match="deadline"):
+        run(fetch_skill("https://skills.example/slow/SKILL.md", deadline_seconds=0.05))
@@ -0,0 +1,107 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.services.skills import limits
+from devplacepy.services.skills.parser import SkillFormatError, parse_skill_md
+
+
+def skill(frontmatter: str, body: str = "# Body\n\nDo the thing.") -> str:
+    return f"---\n{frontmatter}\n---\n{body}"
+
+
+def test_minimal_skill_parses():
+    parsed = parse_skill_md(skill("name: my-skill\ndescription: Does a thing."))
+    assert parsed.name == "my-skill"
+    assert parsed.description == "Does a thing."
+    assert parsed.body.startswith("# Body")
+    assert parsed.metadata == {}
+    assert parsed.warnings == ()
+
+
+def test_all_known_fields_parse():
+    parsed = parse_skill_md(
+        skill(
+            "name: full\n"
+            "description: Full skill\n"
+            "license: MIT\n"
+            "compatibility: any agent\n"
+            "allowed-tools: Read Write\n"
+            "metadata:\n"
+            "  author: someone\n"
+            "  version: '1.0'"
+        )
+    )
+    assert parsed.license == "MIT"
+    assert parsed.compatibility == "any agent"
+    assert parsed.allowed_tools == "Read Write"
+    assert parsed.metadata == {"author": "someone", "version": "1.0"}
+
+
+def test_block_scalars():
+    folded = parse_skill_md(skill("name: a\ndescription: >\n  first line\n  second line"))
+    assert folded.description == "first line second line"
+    literal = parse_skill_md(skill("name: a\ndescription: |\n  first\n  second"))
+    assert literal.description == "first\nsecond"
+
+
+def test_quoted_values_and_inline_comments():
+    parsed = parse_skill_md(skill('name: a\ndescription: "Quoted: value" '))
+    assert parsed.description == "Quoted: value"
+    parsed = parse_skill_md(skill("name: a\ndescription: plain # a comment"))
+    assert parsed.description == "plain"
+
+
+def test_unknown_key_is_ignored_with_a_warning():
+    parsed = parse_skill_md(skill("name: a\ndescription: d\nfuture-key: x"))
+    assert parsed.warnings == ("Ignored unknown frontmatter key 'future-key'.",)
+
+
+def test_bom_and_crlf_are_accepted():
+    text = "---\r\nname: a\r\ndescription: d\r\n---\r\nBody"
+    assert parse_skill_md(text).body == "Body"
+
+
+@pytest.mark.parametrize(
+    "text,fragment",
+    [
+        ("name: a\ndescription: d", "no YAML frontmatter"),
+        ("---\nname: a\ndescription: d\n", "not closed"),
+        (skill("description: d"), "'name' is required"),
+        (skill("name: a"), "'description' is required"),
+        (skill("name: A_B\ndescription: d"), "lowercase letters"),
+        (skill("name: -a\ndescription: d"), "lowercase letters"),
+        (skill("name: a--b\ndescription: d"), "lowercase letters"),
+        (skill("name: a\ndescription: d\nname: b"), "duplicate frontmatter key"),
+        (skill("name: a\ndescription: [x]"), "flow collections"),
+        (skill("name: a\ndescription: d\nmetadata: x"), "indented map"),
+        (skill("name: a\ndescription: d\nmetadata:\n  a: 1\n    b: 2"), "flat map"),
+        (skill("name: a\ndescription: d\nmetadata:\n  a: 1\n  a: 2"), "duplicate metadata key"),
+        (skill("name: a\ndescription: \"unterminated"), "unterminated"),
+        (skill("name: a\ndescription: d\n\tbad: x"), "tab"),
+        (skill("name: a\ndescription: d") + "\x00", "NUL"),
+    ],
+)
+def test_malformed_skills_are_rejected_with_a_reason(text, fragment):
+    with pytest.raises(SkillFormatError) as error:
+        parse_skill_md(text)
+    assert fragment in str(error.value)
+
+
+def test_overlong_fields_are_rejected():
+    with pytest.raises(SkillFormatError):
+        parse_skill_md(skill(f"name: {'a' * (limits.NAME_MAX_CHARS + 1)}\ndescription: d"))
+    with pytest.raises(SkillFormatError):
+        parse_skill_md(skill(f"name: a\ndescription: {'d' * (limits.DESCRIPTION_MAX_CHARS + 1)}"))
+
+
+def test_metadata_entry_cap():
+    entries = "\n".join(f"  k{i}: v" for i in range(limits.METADATA_MAX_ENTRIES + 1))
+    with pytest.raises(SkillFormatError):
+        parse_skill_md(skill(f"name: a\ndescription: d\nmetadata:\n{entries}"))
+
+
+def test_frontmatter_scan_is_bounded():
+    padding = "\n".join("# c" for _ in range(limits.FRONTMATTER_MAX_LINES + 5))
+    with pytest.raises(SkillFormatError):
+        parse_skill_md(f"---\nname: a\ndescription: d\n{padding}\n---\nBody")