# retoor import copy import pytest from devplacepy.services.community.discoverer import CommunityDiscoverer from devplacepy.services.community.schemacheck import sample_limits, validate_schema from tests.conftest import run_async from tests.unit.services.community.fakes import FakeFetcher, html_community, json_community @pytest.fixture(scope="module") def json_schema(): return run_async(CommunityDiscoverer(FakeFetcher(json_community())).discover("https://forum.test/")).schema @pytest.fixture(scope="module") def html_schema(): return run_async(CommunityDiscoverer(FakeFetcher(html_community())).discover("https://board.test/")).schema def test_learned_schemas_are_valid_by_construction(json_schema, html_schema): assert validate_schema(json_schema) == [] assert validate_schema(html_schema) == [] @pytest.mark.parametrize("bad", [None, [], "text", 4]) def test_the_top_level_must_be_an_object(bad): assert validate_schema(bad) == ["the schema must be a JSON object"] def mutated(schema, change): copied = copy.deepcopy(schema) change(copied) return validate_schema(copied) def test_every_structural_mistake_is_named_with_its_path(json_schema, html_schema): assert any("origin" in item for item in mutated(json_schema, lambda s: s.update(origin="Not A Host!"))) assert any("community" in item for item in mutated(json_schema, lambda s: s.update(community=""))) assert any("feeds" in item for item in mutated(json_schema, lambda s: s.update(feeds=[]))) assert any("feeds: at most" in item for item in mutated(json_schema, lambda s: s.update(feeds=[copy.deepcopy(s["feeds"][0]) | {"id": f"f{n}"} for n in range(9)]))) assert any("used twice" in item for item in mutated(json_schema, lambda s: s.update(feeds=[s["feeds"][0], copy.deepcopy(s["feeds"][0])]))) assert any("feeds[0].url" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(url="ftp://x"))) assert any("feeds[0].type" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(type="soap"))) assert any("post.fields.id" in item for item in mutated(json_schema, lambda s: s["feeds"][0]["post"]["fields"].pop("id"))) assert any("content or title" in item for item in mutated(json_schema, lambda s: [s["feeds"][0]["post"]["fields"].pop(k) for k in ("content", "title")])) assert any("$" in item for item in mutated(json_schema, lambda s: s["feeds"][0]["post"]["fields"].update(id=["id"]))) assert any("item_selector" in item for item in mutated(html_schema, lambda s: s["feeds"][0].pop("item_selector"))) assert any("unsupported" in item for item in mutated(html_schema, lambda s: s["feeds"][0]["post"]["fields"].update(title=["h2::bogus"]))) assert any("selector" in item for item in mutated(html_schema, lambda s: s["feeds"][0]["post"]["fields"].update(title=["h2[unclosed"]))) def test_comment_and_pagination_specs_are_checked(json_schema, html_schema): assert any("comments.mode" in item for item in mutated(json_schema, lambda s: s["feeds"][0]["comments"].update(mode="telepathy"))) assert any("list_path" in item for item in mutated(json_schema, lambda s: s["feeds"][0]["comments"].pop("list_path"))) assert any("url_template" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(comments={"mode": "url", "fields": {"id": ["$.id"], "content": ["$.b"]}, "url_template": "https://x/y"}))) assert any("item_selector" in item for item in mutated(html_schema, lambda s: s["feeds"][0]["comments"].pop("item_selector"))) assert any("pagination.style" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(pagination={"style": "psychic"}))) assert any("param" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(pagination={"style": "page"}))) assert any("per_page" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(pagination={"style": "offset", "param": "o"}))) assert mutated(json_schema, lambda s: s["feeds"][0].update(pagination={"style": "none"})) == [] assert any("pagination.next" in item for item in mutated(html_schema, lambda s: s["feeds"][0]["comments"].update(pagination={"style": "next_link", "next": "a[::attr(href)"}))) assert mutated(html_schema, lambda s: s["feeds"][0]["comments"].update(pagination={"style": "page_param", "param": "cp"})) == [] def test_limits_have_ceilings_and_a_politeness_floor(json_schema): assert any("max_posts" in item for item in mutated(json_schema, lambda s: s["limits"].update(max_posts=100000))) assert any("delay_seconds" in item and "polite" in item for item in mutated(json_schema, lambda s: s["limits"].update(delay_seconds=0.01))) assert any("unknown limit" in item for item in mutated(json_schema, lambda s: s["limits"].update(turbo=1))) assert any("positive" in item for item in mutated(json_schema, lambda s: s["limits"].update(retries=-1))) assert any("positive" in item for item in mutated(json_schema, lambda s: s["limits"].update(retries=True))) assert mutated(json_schema, lambda s: s["limits"].update(delay_seconds=2.5, max_posts=10)) == [] def test_login_specs(json_schema): assert any("login.mode" in item for item in mutated(json_schema, lambda s: s.update(login={"mode": "magic"}))) assert any("username_field" in item for item in mutated(json_schema, lambda s: s.update(login={"mode": "json", "url": "https://x/l"}))) assert mutated(json_schema, lambda s: s.update(login={"mode": "json", "url": "https://x/l", "username_field": "u", "password_field": "p", "token_path": "$.t"})) == [] def test_the_sample_run_is_small_and_polite(json_schema): limits = sample_limits(json_schema) assert limits["max_pages"] == 1 and limits["max_comment_pages"] == 1 and limits["max_posts"] <= 12 assert limits["delay_seconds"] >= 1.0 fast = copy.deepcopy(json_schema) fast["limits"]["delay_seconds"] = 5.0 assert sample_limits(fast)["delay_seconds"] == 5.0