devplacepy/tests/unit/services/skills/fetcher.py

# retoor <retoor@molodetz.nl>

import pytest

from devplacepy.services.skills import limits
from devplacepy.services.skills.fetcher import (
    SkillFetchError,
    classify_candidate,
    decode_text,
    directory_url,
    extract_candidates,
    normalize_input_url,
    origin_of,
)
from devplacepy.services.skills.parser import SkillFormatError


def test_normalize_adds_scheme_and_strips_fragment():
    assert normalize_input_url("example.org/skill#frag") == "https://example.org/skill"
    assert normalize_input_url("  http://example.org/a/SKILL.md ") == "http://example.org/a/SKILL.md"


@pytest.mark.parametrize(
    "raw",
    ["", "   ", "ftp://example.org/x", "https://user:pw@example.org/x", "x" * (limits.URL_MAX_CHARS + 1)],
)
def test_normalize_rejects_bad_urls(raw):
    with pytest.raises(SkillFetchError):
        normalize_input_url(raw)


@pytest.mark.parametrize(
    "url,expected",
    [
        ("https://e.org/a/SKILL.md", "https://e.org/a/"),
        ("https://e.org/a/index.html", "https://e.org/a/"),
        ("https://e.org/a", "https://e.org/a/"),
        ("https://e.org/a/", "https://e.org/a/"),
        ("https://e.org", "https://e.org/"),
    ],
)
def test_directory_url(url, expected):
    assert directory_url(url) == expected


def test_origin_uses_default_ports():
    assert origin_of("https://Example.org/x") == ("https", "example.org", 443)
    assert origin_of("http://example.org:8080/x") == ("http", "example.org", 8080)


def test_candidates_come_from_links_and_code_spans_once():
    body = (
        "See [the script](scripts/run.py) and `references/guide.md`.\n"
        "Again [dup](scripts/run.py#anchor) and [other](docs/readme.md) and `not a path`."
    )
    assert extract_candidates(body) == ["scripts/run.py", "references/guide.md"]


@pytest.mark.parametrize(
    "candidate,accepted",
    [
        ("scripts/run.py", True),
        ("./references/guide.md", True),
        ("assets/templates/SPEC.md", True),
        ("assets/img/hero.png", False),
        ("../scripts/run.py", False),
        ("scripts/../../etc/passwd", False),
        ("scripts/%2e%2e/x.py", False),
        ("scripts/a.py?x=1", False),
        ("/scripts/run.py", False),
        ("https://evil.org/scripts/run.py", False),
        ("scripts\\run.py", False),
        ("other/run.py", False),
        ("scripts", False),
        ("scripts/sp ace.py", False),
    ],
)
def test_classify_candidate_only_accepts_safe_relative_text_paths(candidate, accepted):
    path, reason = classify_candidate(candidate)
    assert (path is not None) is accepted
    assert (reason == "") is accepted


def test_overlong_path_is_rejected():
    path, reason = classify_candidate("scripts/" + "a" * limits.BUNDLED_PATH_MAX_CHARS + ".py")
    assert path is None and reason


def test_decode_text():
    assert decode_text("héllo".encode("utf-8")) == "héllo"
    with pytest.raises(SkillFormatError):
        decode_text(b"a\x00b")
    with pytest.raises(SkillFormatError):
        decode_text(b"\xff\xfe\xfa")


import ipaddress
import socket
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer

from devplacepy.net_guard import is_blocked_address
from devplacepy.services.skills.fetcher import fetch_skill
from tests.conftest import run_async

SKILL_TEXT = (
    "---\nname: wire-skill\ndescription: Fetched over a real socket.\n---\n"
    "# Wire skill\n\nRead [the guide](references/guide.md) and `references/missing.md`.\n"
)


class Handler(BaseHTTPRequestHandler):
    routes: dict = {}

    def do_GET(self):
        entry = self.routes.get(self.path.split("?", 1)[0])
        if entry is None:
            self.send_response(404)
            self.end_headers()
            return
        status, headers, body = entry
        self.send_response(status)
        for name, value in headers.items():
            self.send_header(name, value)
        self.send_header("Content-Length", str(len(body)))
        self.end_headers()
        self.wfile.write(body)

    def log_message(self, *args):
        return None


def global_address() -> str | None:
    probe = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    try:
        probe.connect(("8.8.8.8", 80))
        address = probe.getsockname()[0]
    except OSError:
        return None
    finally:
        probe.close()
    return None if is_blocked_address(ipaddress.ip_address(address)) else address


needs_global_address = pytest.mark.skipif(
    global_address() is None,
    reason="this host has no globally routable IPv4 address, so a local server cannot pass the SSRF guard without patching it",
)


@pytest.fixture
def origin():
    port_holder = {}

    def text(body, kind="text/markdown"):
        return 200, {"Content-Type": kind}, body.encode()

    Handler.routes = {
        "/ok/": text("<html><body>skill directory</body></html>", "text/html"),
        "/ok/SKILL.md": text(SKILL_TEXT),
        "/ok/references/guide.md": text("# Guide\n\nStep one.\n"),
        "/big/SKILL.md": text("---\nname: big\ndescription: too big\n---\n" + "x" * (limits.SKILL_MD_MAX_BYTES + 10)),
        "/html/SKILL.md": text("<html><body>not a skill</body></html>", "text/html"),
        "/bounce/SKILL.md": (302, {"Location": "PLACEHOLDER"}, b""),
        "/loop/SKILL.md": (302, {"Location": "/loop/SKILL.md"}, b""),
    }
    server = ThreadingHTTPServer(("0.0.0.0", 0), Handler)
    port_holder["port"] = server.server_address[1]
    thread = threading.Thread(target=server.serve_forever, daemon=True)
    thread.start()
    yield port_holder["port"]
    server.shutdown()
    server.server_close()


@needs_global_address
def test_a_skill_is_fetched_over_a_real_socket_with_its_bundled_file(origin):
    address = global_address()
    fetched = run_async(fetch_skill(f"http://{address}:{origin}/ok/SKILL.md"))
    assert fetched.parsed.name == "wire-skill" and fetched.source_url.endswith("/ok/SKILL.md")
    assert fetched.files == {"references/guide.md": "# Guide\n\nStep one.\n"}
    assert [(item.path, item.reason) for item in fetched.skipped] == [("references/missing.md", "the server returned 404")]


@needs_global_address
def test_the_directory_address_resolves_to_the_skill_file(origin):
    address = global_address()
    fetched = run_async(fetch_skill(f"http://{address}:{origin}/ok/"))
    assert fetched.parsed.name == "wire-skill"


@needs_global_address
def test_loopback_is_refused_even_though_the_server_is_right_there(origin):
    with pytest.raises(SkillFetchError, match="private or local"):
        run_async(fetch_skill(f"http://127.0.0.1:{origin}/ok/SKILL.md"))


@needs_global_address
def test_a_redirect_to_loopback_is_refused_on_the_second_hop(origin):
    address = global_address()
    Handler.routes["/bounce/SKILL.md"] = (302, {"Location": f"http://127.0.0.1:{origin}/ok/SKILL.md"}, b"")
    with pytest.raises(SkillFetchError, match="private or local"):
        run_async(fetch_skill(f"http://{address}:{origin}/bounce/SKILL.md"))


@needs_global_address
def test_a_redirect_loop_ends_with_a_clean_error(origin):
    address = global_address()
    with pytest.raises(SkillFetchError):
        run_async(fetch_skill(f"http://{address}:{origin}/loop/SKILL.md"))


@needs_global_address
def test_an_oversize_skill_is_cut_at_the_limit_and_html_is_rejected(origin):
    address = global_address()
    with pytest.raises(SkillFetchError, match="byte limit"):
        run_async(fetch_skill(f"http://{address}:{origin}/big/SKILL.md"))
    with pytest.raises(SkillFetchError, match="HTML page"):
        run_async(fetch_skill(f"http://{address}:{origin}/html/SKILL.md"))


@needs_global_address
def test_a_missing_skill_reports_the_status(origin):
    address = global_address()
    with pytest.raises(SkillFetchError, match="404"):
        run_async(fetch_skill(f"http://{address}:{origin}/nothing/SKILL.md"))