chore: reorganize test files into domain-specific subdirectories

2026-06-13 14:32:33 +00:00
parent 8eac4cafba
commit eccec02336
268 changed files with 20634 additions and 9206 deletions
@@ -0,0 +1,212 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+from datetime import datetime, timezone
+from pathlib import Path
+import pytest
+from devplacepy.database import init_db, get_table, refresh_snapshot
+from devplacepy import project_files
+from devplacepy.services.jobs import queue
+from devplacepy.services.jobs.fork_service import ForkService
+from tests.conftest import run_async
+@pytest.fixture(autouse=True)
+def _init_db_fork_jobs():
+    init_db()
+    yield
+@pytest.fixture
+def fork_env(tmp_path, monkeypatch):
+    monkeypatch.setattr(
+        "devplacepy.services.jobs.fork_service.STAGING_DIR", tmp_path / "staging"
+    )
+    monkeypatch.setattr("devplacepy.project_files.PROJECT_FILES_DIR", tmp_path / "pf")
+    yield tmp_path
+    jobs = get_table("jobs")
+    for row in list(jobs.find(kind="fork")):
+        jobs.delete(uid=row["uid"])
+    projects = get_table("projects")
+    files = get_table("project_files")
+    for project in list(projects.find()):
+        if str(project.get("user_uid", "")).startswith("forktest-owner"):
+            for node in list(files.find(project_uid=project["uid"])):
+                files.delete(uid=node["uid"])
+            projects.delete(uid=project["uid"])
+    for node in list(files.find()):
+        if str(node.get("project_uid", "")).startswith("forktest"):
+            files.delete(uid=node["uid"])
+    forks = get_table("project_forks")
+    for relation in list(forks.find()):
+        if str(relation.get("forked_by_uid", "")).startswith("forktest-owner"):
+            forks.delete(uid=relation["uid"])
+    users = get_table("users")
+    for user in list(users.find()):
+        if str(user.get("uid", "")).startswith("forktest-owner"):
+            users.delete(uid=user["uid"])
+_counter_fork_jobs = [0]
+def _make_source_project(*, is_private=False, binary=False):
+    _counter_fork_jobs[0] += 1
+    pid = f"forktest-{_counter_fork_jobs[0]}"
+    owner_uid = f"forktest-owner-{_counter_fork_jobs[0]}"
+    user = {"uid": owner_uid, "username": f"forktester{_counter_fork_jobs[0]}"}
+    get_table("users").insert(
+        {"uid": owner_uid, "username": user["username"], "xp": 0, "level": 1}
+    )
+    get_table("projects").insert(
+        {
+            "uid": pid,
+            "user_uid": owner_uid,
+            "slug": f"{pid}-source",
+            "title": "Source Project",
+            "description": "the original",
+            "project_type": "software",
+            "platforms": "linux",
+            "status": "Released",
+            "is_private": 1 if is_private else 0,
+            "read_only": 0,
+            "stars": 0,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        }
+    )
+    project_files.write_text_file(pid, user, "README.md", "# hello\nworld")
+    project_files.write_text_file(pid, user, "src/app.py", "print(1)\n")
+    if binary:
+        project_files.store_upload(pid, user, "assets", "logo.bin", bytes(range(256)))
+    return pid, owner_uid
+def _tree(directory):
+    root = Path(directory)
+    out = {}
+    for path in sorted(root.rglob("*")):
+        if path.is_file():
+            out[path.relative_to(root).as_posix()] = path.read_bytes()
+    return out
+def _process_fork_jobs():
+    async def drive():
+        svc = ForkService()
+        for _ in range(400):
+            await svc.run_once()
+            refresh_snapshot()
+            pending = [
+                r
+                for r in get_table("jobs").find(kind="fork")
+                if r["status"] in ("pending", "running")
+            ]
+            if not pending and not svc._inflight:
+                return
+            await asyncio.sleep(0.05)
+
+    run_async(drive())
+def _enqueue(source_uid, owner_uid, title="My Fork"):
+    return queue.enqueue(
+        "fork",
+        {"source_project_uid": source_uid, "title": title, "forked_by_uid": owner_uid},
+        "user",
+        owner_uid,
+        title,
+    )
+
+
+def test_enqueue_creates_pending_job(fork_env):
+    uid = _enqueue("p", "forktest-owner-x")
+    job = queue.get_job(uid)
+    assert job["status"] == "pending"
+    assert job["kind"] == "fork"
+    assert job["payload"]["source_project_uid"] == "p"
+
+
+def test_process_creates_fork_with_files(fork_env, tmp_path):
+    pid, owner_uid = _make_source_project(binary=True)
+    uid = _enqueue(pid, owner_uid, title="Forked Copy")
+    _process_fork_jobs()
+    job = queue.get_job(uid)
+    assert job["status"] == "done"
+    result = job["result"]
+    new_uid = result["project_uid"]
+    forked = get_table("projects").find_one(uid=new_uid)
+    assert forked is not None
+    assert forked["user_uid"] == owner_uid
+    assert forked["title"] == "Forked Copy"
+    assert result["project_url"] == f"/projects/{forked['slug']}"
+
+    source_dir = tmp_path / "exp_source"
+    fork_dir = tmp_path / "exp_fork"
+    project_files.export_to_dir(pid, "", source_dir)
+    project_files.export_to_dir(new_uid, "", fork_dir)
+    assert _tree(fork_dir) == _tree(source_dir)
+
+
+def test_fork_relation_direction(fork_env):
+    pid, owner_uid = _make_source_project()
+    uid = _enqueue(pid, owner_uid)
+    _process_fork_jobs()
+    new_uid = queue.get_job(uid)["result"]["project_uid"]
+    relation = get_table("project_forks").find_one(forked_project_uid=new_uid)
+    assert relation is not None
+    assert relation["source_project_uid"] == pid
+    assert relation["forked_project_uid"] == new_uid
+    assert relation["forked_by_uid"] == owner_uid
+
+
+def test_fork_copies_binary_file(fork_env, tmp_path):
+    pid, owner_uid = _make_source_project(binary=True)
+    uid = _enqueue(pid, owner_uid)
+    _process_fork_jobs()
+    new_uid = queue.get_job(uid)["result"]["project_uid"]
+    fork_dir = tmp_path / "binfork"
+    project_files.export_to_dir(new_uid, "", fork_dir)
+    assert (fork_dir / "assets" / "logo.bin").read_bytes() == bytes(range(256))
+
+
+def test_fork_preserves_private_flag(fork_env):
+    pid, owner_uid = _make_source_project(is_private=True)
+    uid = _enqueue(pid, owner_uid)
+    _process_fork_jobs()
+    new_uid = queue.get_job(uid)["result"]["project_uid"]
+    assert get_table("projects").find_one(uid=new_uid)["is_private"] == 1
+
+
+def test_missing_source_fails_job_without_orphan(fork_env):
+    before = {p["uid"] for p in get_table("projects").find()}
+    uid = _enqueue("does-not-exist", "forktest-owner-missing")
+    _process_fork_jobs()
+    job = queue.get_job(uid)
+    assert job["status"] == "failed"
+    assert "source project not found" in job["error"].lower()
+    after = {p["uid"] for p in get_table("projects").find()}
+    assert before == after
+
+
+def test_cleanup_keeps_project(fork_env):
+    pid, owner_uid = _make_source_project()
+    uid = _enqueue(pid, owner_uid)
+    _process_fork_jobs()
+    job = queue.get_job(uid)
+    new_uid = job["result"]["project_uid"]
+    ForkService().cleanup(job)
+    assert get_table("projects").find_one(uid=new_uid) is not None
+
+
+def test_retention_sweep_keeps_project(fork_env):
+    pid, owner_uid = _make_source_project()
+    uid = _enqueue(pid, owner_uid)
+    _process_fork_jobs()
+    new_uid = queue.get_job(uid)["result"]["project_uid"]
+    get_table("jobs").update(
+        {"uid": uid, "expires_at": "2000-01-01T00:00:00+00:00"}, ["uid"]
+    )
+    svc = ForkService()
+    run_async(svc.run_once())
+    refresh_snapshot()
+    assert queue.get_job(uid) is None
+    assert get_table("projects").find_one(uid=new_uid) is not None
+
+
+def test_orphan_running_recovered_on_enable(fork_env):
+    uid = _enqueue("p", "forktest-owner-o")
+    get_table("jobs").update(
+        {"uid": uid, "status": "running", "started_at": "2020-01-01T00:00:00+00:00"},
+        ["uid"],
+    )
+    svc = ForkService()
+    run_async(svc.on_enable())
+    job = queue.get_job(uid)
+    assert job["status"] == "pending"
+    assert job["retry_count"] == 1
@@ -0,0 +1,401 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+import zipfile
+import zlib
+from pathlib import Path
+import pytest
+from devplacepy.database import init_db, get_table, refresh_snapshot
+from devplacepy import project_files
+from devplacepy.project_files import ProjectFileError
+from devplacepy.services.jobs import queue
+from devplacepy.services.jobs.zip_service import ZipService
+from devplacepy.services.jobs import zip_worker
+from tests.conftest import run_async
+@pytest.fixture(autouse=True)
+def _init_db_zip_jobs():
+    init_db()
+    yield
+@pytest.fixture
+def zip_env(tmp_path, monkeypatch):
+    monkeypatch.setattr(
+        "devplacepy.services.jobs.zip_service.ZIPS_DIR", tmp_path / "zips"
+    )
+    monkeypatch.setattr(
+        "devplacepy.services.jobs.zip_service.STAGING_DIR", tmp_path / "staging"
+    )
+    monkeypatch.setattr("devplacepy.project_files.PROJECT_FILES_DIR", tmp_path / "pf")
+    yield tmp_path
+    jobs = get_table("jobs")
+    for row in list(jobs.find(kind="zip")):
+        jobs.delete(uid=row["uid"])
+    files = get_table("project_files")
+    for row in list(files.find()):
+        if str(row.get("project_uid", "")).startswith("ziptest"):
+            files.delete(uid=row["uid"])
+_pid_counter = [0]
+def _make_project(text=None, binary=False):
+    _pid_counter[0] += 1
+    pid = f"ziptest-{_pid_counter[0]}"
+    user = {"uid": f"ziptest-owner-{_pid_counter[0]}"}
+    project_files.write_text_file(pid, user, "README.md", "# hello\nworld")
+    project_files.write_text_file(pid, user, "src/app.py", "print(1)\n")
+    if binary:
+        project_files.store_upload(pid, user, "assets", "logo.bin", bytes(range(256)))
+    return pid, user
+def _process_zip_jobs():
+    async def drive():
+        svc = ZipService()
+        for _ in range(400):
+            await svc.run_once()
+            refresh_snapshot()
+            pending = [
+                r
+                for r in get_table("jobs").find(kind="zip")
+                if r["status"] in ("pending", "running")
+            ]
+            if not pending and not svc._inflight:
+                return
+            await asyncio.sleep(0.05)
+
+    run_async(drive())
+
+
+def test_enqueue_creates_pending_job(zip_env):
+    uid = queue.enqueue("zip", {"a": 1}, "user", "u1", "Name")
+    job = queue.get_job(uid)
+    assert job["status"] == "pending"
+    assert job["kind"] == "zip"
+    assert job["payload"] == {"a": 1}
+    assert job["result"] == {}
+    assert job["owner_kind"] == "user" and job["owner_id"] == "u1"
+    assert job["created_at"] and job["expires_at"] == ""
+
+
+def test_get_job_missing_returns_none(zip_env):
+    assert queue.get_job("does-not-exist") is None
+
+
+def test_touch_job_extends_expiry(zip_env):
+    uid = queue.enqueue("zip", {}, "guest", "g1", "x")
+    queue.touch_job(uid, 3600)
+    job = queue.get_job(uid)
+    assert job["last_accessed_at"] and job["expires_at"]
+
+
+def test_list_jobs_filters(zip_env):
+    a = queue.enqueue("zip", {}, "user", "owner-A", "a")
+    queue.enqueue("zip", {}, "user", "owner-B", "b")
+    only_a = queue.list_jobs(kind="zip", owner=("user", "owner-A"))
+    assert [j["uid"] for j in only_a] == [a]
+
+
+def test_zip_worker_stats_and_crc(tmp_path):
+    src = tmp_path / "src"
+    (src / "sub").mkdir(parents=True)
+    (src / "a.txt").write_text("hello")
+    (src / "sub" / "b.txt").write_text("world")
+    out = tmp_path / "out.zip"
+    stats = zip_worker._build(str(src), str(out))
+    assert stats["file_count"] == 2
+    assert stats["dir_count"] == 1
+    assert stats["bytes_in"] == 10
+    assert stats["bytes_out"] == out.stat().st_size
+    expected_crc = 0
+    with open(out, "rb") as handle:
+        for chunk in iter(lambda: handle.read(65536), b""):
+            expected_crc = zlib.crc32(chunk, expected_crc)
+    assert stats["crc32"] == (expected_crc & 0xFFFFFFFF)
+    assert sorted(zipfile.ZipFile(out).namelist()) == ["a.txt", "sub/", "sub/b.txt"]
+
+
+def test_process_whole_project(zip_env):
+    pid, _ = _make_project(binary=True)
+    uid = queue.enqueue(
+        "zip",
+        {"source": {"type": "project_tree", "project_uid": pid, "path": ""}},
+        "user",
+        "u",
+        "Proj",
+    )
+    _process_zip_jobs()
+    job = queue.get_job(uid)
+    assert job["status"] == "done"
+    assert job["duration_ms"] >= 0
+    assert job["expires_at"]
+    result = job["result"]
+    names = sorted(zipfile.ZipFile(result["local_path"]).namelist())
+    assert names == ["README.md", "assets/", "assets/logo.bin", "src/", "src/app.py"]
+    assert result["file_count"] == 3 and result["dir_count"] == 2
+    assert job["bytes_out"] == Path(result["local_path"]).stat().st_size
+
+
+def test_process_subtree_folder(zip_env):
+    pid, _ = _make_project()
+    uid = queue.enqueue(
+        "zip",
+        {"source": {"type": "project_tree", "project_uid": pid, "path": "src"}},
+        "user",
+        "u",
+        "src",
+    )
+    _process_zip_jobs()
+    result = queue.get_job(uid)["result"]
+    assert sorted(zipfile.ZipFile(result["local_path"]).namelist()) == [
+        "src/",
+        "src/app.py",
+    ]
+
+
+def test_process_single_file(zip_env):
+    pid, _ = _make_project()
+    uid = queue.enqueue(
+        "zip",
+        {"source": {"type": "project_tree", "project_uid": pid, "path": "src/app.py"}},
+        "user",
+        "u",
+        "app.py",
+    )
+    _process_zip_jobs()
+    result = queue.get_job(uid)["result"]
+    assert zipfile.ZipFile(result["local_path"]).namelist() == ["app.py"]
+
+
+def test_final_name_format_and_strip_zip(zip_env):
+    pid, _ = _make_project()
+    uid = queue.enqueue(
+        "zip",
+        {"source": {"type": "project_tree", "project_uid": pid, "path": ""}},
+        "user",
+        "u",
+        "My Project.ZIP",
+    )
+    _process_zip_jobs()
+    result = queue.get_job(uid)["result"]
+    name = result["final_name"]
+    assert name.endswith(".my-project.zip")
+    crc_part = name.split(".", 1)[0]
+    assert len(crc_part) == 8 and int(crc_part, 16) >= 0
+    assert crc_part == result["crc32"]
+
+
+def test_blank_preferred_name_defaults(zip_env):
+    pid, _ = _make_project()
+    uid = queue.enqueue(
+        "zip",
+        {"source": {"type": "project_tree", "project_uid": pid, "path": ""}},
+        "user",
+        "u",
+        "",
+    )
+    _process_zip_jobs()
+    assert queue.get_job(uid)["result"]["final_name"].endswith(".download.zip")
+
+
+def test_identical_content_replaces_same_name(zip_env):
+    pid, _ = _make_project()
+    payload = {"source": {"type": "project_tree", "project_uid": pid, "path": ""}}
+    uid1 = queue.enqueue("zip", payload, "user", "u", "thing")
+    _process_zip_jobs()
+    name1 = queue.get_job(uid1)["result"]["final_name"]
+    uid2 = queue.enqueue("zip", payload, "user", "u", "thing")
+    _process_zip_jobs()
+    name2 = queue.get_job(uid2)["result"]["final_name"]
+    assert name1 == name2
+
+
+def test_cleanup_removes_artifact(zip_env):
+    pid, _ = _make_project()
+    uid = queue.enqueue(
+        "zip",
+        {"source": {"type": "project_tree", "project_uid": pid, "path": ""}},
+        "user",
+        "u",
+        "thing",
+    )
+    _process_zip_jobs()
+    job = queue.get_job(uid)
+    path = Path(job["result"]["local_path"])
+    assert path.is_file()
+    ZipService().cleanup(job)
+    assert not path.exists()
+
+
+def test_unsupported_source_fails_job(zip_env):
+    uid = queue.enqueue("zip", {"source": {"type": "evil"}}, "user", "u", "x")
+    _process_zip_jobs()
+    job = queue.get_job(uid)
+    assert job["status"] == "failed"
+    assert "unsupported" in job["error"].lower()
+
+
+def test_normalize_path_rejects_parent(zip_env):
+    with pytest.raises(ProjectFileError):
+        project_files.normalize_path("a/../../b")
+    with pytest.raises(ProjectFileError):
+        project_files.normalize_path("../etc/passwd")
+
+
+def test_export_blocks_malicious_db_path(zip_env, tmp_path):
+    pid = "ziptest-evil"
+    get_table("project_files").insert(
+        {
+            "uid": "evil-node",
+            "project_uid": pid,
+            "user_uid": "u",
+            "path": "../escape.txt",
+            "name": "escape.txt",
+            "parent_path": "",
+            "type": "file",
+            "content": "pwned",
+            "is_binary": 0,
+            "stored_name": None,
+            "directory": None,
+            "mime_type": "text/plain",
+            "size": 5,
+            "created_at": "x",
+            "updated_at": "x",
+        }
+    )
+    dest = tmp_path / "dest"
+    with pytest.raises(ProjectFileError):
+        project_files.export_to_dir(pid, "", dest)
+    assert not (tmp_path / "escape.txt").exists()
+
+
+def test_traversal_payload_marks_job_failed(zip_env):
+    pid = "ziptest-evil2"
+    get_table("project_files").insert(
+        {
+            "uid": "evil-node2",
+            "project_uid": pid,
+            "user_uid": "u",
+            "path": "../../escape2.txt",
+            "name": "escape2.txt",
+            "parent_path": "",
+            "type": "file",
+            "content": "pwned",
+            "is_binary": 0,
+            "stored_name": None,
+            "directory": None,
+            "mime_type": "text/plain",
+            "size": 6,
+            "created_at": "x",
+            "updated_at": "x",
+        }
+    )
+    uid = queue.enqueue(
+        "zip",
+        {"source": {"type": "project_tree", "project_uid": pid, "path": ""}},
+        "user",
+        "u",
+        "evil",
+    )
+    _process_zip_jobs()
+    assert queue.get_job(uid)["status"] == "failed"
+
+
+def test_orphan_running_recovered_on_enable(zip_env):
+    uid = queue.enqueue(
+        "zip",
+        {"source": {"type": "project_tree", "project_uid": "p", "path": ""}},
+        "user",
+        "u",
+        "n",
+    )
+    get_table("jobs").update(
+        {"uid": uid, "status": "running", "started_at": "2020-01-01T00:00:00+00:00"},
+        ["uid"],
+    )
+    svc = ZipService()
+    run_async(svc.on_enable())
+    job = queue.get_job(uid)
+    assert job["status"] == "pending"
+    assert job["retry_count"] == 1
+
+
+def test_orphan_exceeds_retry_limit_fails(zip_env):
+    uid = queue.enqueue(
+        "zip",
+        {"source": {"type": "project_tree", "project_uid": "p", "path": ""}},
+        "user",
+        "u",
+        "n",
+    )
+    get_table("jobs").update(
+        {
+            "uid": uid,
+            "status": "running",
+            "retry_count": 3,
+            "started_at": "2020-01-01T00:00:00+00:00",
+        },
+        ["uid"],
+    )
+    svc = ZipService()
+    run_async(svc.on_enable())
+    assert queue.get_job(uid)["status"] == "failed"
+
+
+def test_retention_sweep_deletes_expired(zip_env, tmp_path):
+    artifact = tmp_path / "old.zip"
+    artifact.write_bytes(b"PK\x05\x06" + b"\x00" * 18)
+    uid = queue.enqueue("zip", {}, "user", "u", "old")
+    get_table("jobs").update(
+        {
+            "uid": uid,
+            "status": "done",
+            "result": '{"local_path": "%s"}' % artifact.as_posix(),
+            "expires_at": "2000-01-01T00:00:00+00:00",
+        },
+        ["uid"],
+    )
+    svc = ZipService()
+    run_async(svc.run_once())
+    refresh_snapshot()
+    assert queue.get_job(uid) is None
+    assert not artifact.exists()
+
+
+def test_retention_keeps_unexpired(zip_env, tmp_path):
+    uid = queue.enqueue("zip", {}, "user", "u", "fresh")
+    get_table("jobs").update(
+        {
+            "uid": uid,
+            "status": "done",
+            "result": "{}",
+            "expires_at": "2999-01-01T00:00:00+00:00",
+        },
+        ["uid"],
+    )
+    svc = ZipService()
+    run_async(svc.run_once())
+    refresh_snapshot()
+    assert queue.get_job(uid) is not None
+
+
+def test_max_concurrent_caps_inflight(zip_env, monkeypatch):
+    for _ in range(4):
+        queue.enqueue(
+            "zip",
+            {"source": {"type": "project_tree", "project_uid": "p", "path": ""}},
+            "user",
+            "u",
+            "n",
+        )
+
+    async def fake_process(self, job):
+        await asyncio.sleep(0.2)
+        return {}
+
+    monkeypatch.setattr(ZipService, "process", fake_process)
+    monkeypatch.setattr(ZipService, "max_concurrent", lambda self: 2)
+
+    async def check():
+        svc = ZipService()
+        svc._refill()
+        count = len(svc._inflight)
+        for entry in svc._inflight.values():
+            entry["task"].cancel()
+        return count
+
+    assert run_async(check()) == 2