forked from retoor/devplacepy
sync_workspace (user-triggered) and the reconciler's sync_bidirectional_sync could run concurrently for the same project, and store_upload's read-then- write on a changed path meant two racing imports each wrote their own blob while only one ever got referenced - the loser leaked forever. Combined with no build-artifact exclusion, an actively-compiling workspace hit this constantly and leaked 5.9M orphan blobs (~96GB) in production before it was caught. Closes it at the root: api._sync_dir_bidirectional_locked serializes both call sites per-project (non-blocking - a project already mid-sync is simply skipped until the next tick), and IMPORT_SKIP_NAMES/IMPORT_SKIP_EXTENSIONS keep build output (build/, dist/, *.o, *.pyc, ...) out of the walk entirely. Recovering what already leaked is a separate concern: a new CLI subcommand (plus matching make targets) sweeps soft-deleted attachment/project-file blobs and any blob with zero DB reference at all, plus orphaned container workspace directories. run_maintenance_cleanup.sh wraps the existing prune/clear commands for routine disk upkeep. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BWJy6PrMMt5hwWxQwia2rd
205 lines
6.9 KiB
Python
205 lines
6.9 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
import argparse
|
|
from datetime import datetime, timezone
|
|
import pytest
|
|
from devplacepy import cli
|
|
from devplacepy.database import get_table
|
|
from devplacepy.utils import generate_uid
|
|
def _make_user_cli(role="Member"):
|
|
username = f"cli_{generate_uid()[:8]}"
|
|
get_table("users").insert(
|
|
{
|
|
"uid": generate_uid(),
|
|
"username": username,
|
|
"terms_version": "1",
|
|
"email": f"{username}@t.dev",
|
|
"role": role,
|
|
"created_at": datetime.now(timezone.utc).isoformat(),
|
|
}
|
|
)
|
|
return username
|
|
|
|
|
|
def test_role_get_prints_lowercased_role(local_db, capsys):
|
|
username = _make_user_cli(role="Admin")
|
|
cli.cmd_role_get(argparse.Namespace(username=username))
|
|
assert capsys.readouterr().out.strip() == "admin"
|
|
|
|
|
|
def test_role_get_missing_user_exits(local_db):
|
|
with pytest.raises(SystemExit):
|
|
cli.cmd_role_get(argparse.Namespace(username="cli_absent_xyz"))
|
|
|
|
|
|
def test_role_set_updates_role(local_db):
|
|
username = _make_user_cli(role="Member")
|
|
cli.cmd_role_set(argparse.Namespace(username=username, role="admin"))
|
|
user = get_table("users").find_one(username=username)
|
|
assert user["role"] == "Admin"
|
|
|
|
|
|
def test_role_set_invalid_role_exits(local_db):
|
|
username = _make_user_cli()
|
|
with pytest.raises(SystemExit):
|
|
cli.cmd_role_set(argparse.Namespace(username=username, role="superuser"))
|
|
|
|
|
|
def test_role_set_missing_user_exits(local_db):
|
|
with pytest.raises(SystemExit):
|
|
cli.cmd_role_set(argparse.Namespace(username="cli_absent_xyz", role="admin"))
|
|
|
|
|
|
def test_news_clear_deletes_all_tables(local_db, capsys):
|
|
for table in ("news", "news_images", "news_sync"):
|
|
get_table(table).insert({"uid": generate_uid(), "marker": "cli"})
|
|
cli.cmd_news_clear(argparse.Namespace())
|
|
for table in ("news", "news_images", "news_sync"):
|
|
assert get_table(table).count() == 0
|
|
assert "News data cleared" in capsys.readouterr().out
|
|
|
|
|
|
def test_news_sanitize_strips_html(local_db, capsys):
|
|
uid = generate_uid()
|
|
get_table("news").insert(
|
|
{
|
|
"deleted_at": None,
|
|
"deleted_by": None,
|
|
"uid": uid,
|
|
"description": "<b>Bold</b> & clean",
|
|
"content": "<p>Body</p>",
|
|
}
|
|
)
|
|
cli.cmd_news_sanitize(argparse.Namespace())
|
|
row = get_table("news").find_one(uid=uid)
|
|
assert row["description"] == "Bold & clean"
|
|
assert row["content"] == "Body"
|
|
assert "Sanitized" in capsys.readouterr().out
|
|
|
|
|
|
def test_attachments_prune_removes_only_stale_orphans(local_db, capsys):
|
|
old_uid = generate_uid()
|
|
fresh_uid = generate_uid()
|
|
attachments = get_table("attachments")
|
|
attachments.insert(
|
|
{
|
|
"deleted_at": None,
|
|
"deleted_by": None,
|
|
"uid": old_uid,
|
|
"target_type": "",
|
|
"target_uid": "",
|
|
"created_at": "2000-01-01T00:00:00+00:00",
|
|
}
|
|
)
|
|
attachments.insert(
|
|
{
|
|
"deleted_at": None,
|
|
"deleted_by": None,
|
|
"uid": fresh_uid,
|
|
"target_type": "",
|
|
"target_uid": "",
|
|
"created_at": datetime.now(timezone.utc).isoformat(),
|
|
}
|
|
)
|
|
cli.cmd_attachments_prune(argparse.Namespace(hours=24))
|
|
assert attachments.find_one(uid=old_uid) is None
|
|
assert attachments.find_one(uid=fresh_uid) is not None
|
|
|
|
|
|
def test_apikey_get_prints_key(local_db, capsys):
|
|
username = _make_user_cli()
|
|
key = generate_uid()
|
|
get_table("users").update(
|
|
{"uid": get_table("users").find_one(username=username)["uid"], "api_key": key},
|
|
["uid"],
|
|
)
|
|
cli.cmd_apikey_get(argparse.Namespace(username=username))
|
|
assert capsys.readouterr().out.strip() == key
|
|
|
|
|
|
def test_apikey_reset_changes_key(local_db, capsys):
|
|
username = _make_user_cli()
|
|
users = get_table("users")
|
|
users.update(
|
|
{"uid": users.find_one(username=username)["uid"], "api_key": generate_uid()},
|
|
["uid"],
|
|
)
|
|
before = users.find_one(username=username)["api_key"]
|
|
cli.cmd_apikey_reset(argparse.Namespace(username=username))
|
|
printed = capsys.readouterr().out.strip()
|
|
after = users.find_one(username=username)["api_key"]
|
|
assert after != before
|
|
assert printed == after
|
|
|
|
|
|
def test_apikey_backfill_assigns_missing(local_db, capsys):
|
|
users = get_table("users")
|
|
uid = generate_uid()
|
|
users.insert(
|
|
{
|
|
"uid": uid,
|
|
"username": f"cli_{uid[:8]}",
|
|
"terms_version": "1",
|
|
"email": f"{uid[:8]}@t.dev",
|
|
"role": "Member",
|
|
"created_at": datetime.now(timezone.utc).isoformat(),
|
|
}
|
|
)
|
|
cli.cmd_apikey_backfill(argparse.Namespace())
|
|
assert capsys.readouterr().out.strip().startswith("Assigned API keys to")
|
|
assert users.find_one(uid=uid).get("api_key")
|
|
|
|
|
|
def test_main_without_command_exits(monkeypatch):
|
|
monkeypatch.setattr("sys.argv", ["devplace"])
|
|
with pytest.raises(SystemExit):
|
|
cli.main()
|
|
|
|
|
|
def test_main_dispatches_subcommand(local_db, monkeypatch, capsys):
|
|
username = _make_user_cli(role="Admin")
|
|
monkeypatch.setattr("sys.argv", ["devplace", "role", "get", username])
|
|
cli.main()
|
|
assert capsys.readouterr().out.strip() == "admin"
|
|
|
|
|
|
def test_system_prune_reclaims_orphans_across_subsystems(local_db, tmp_path, capsys, monkeypatch):
|
|
from devplacepy import attachments as att
|
|
from devplacepy import project_files as pf
|
|
|
|
monkeypatch.setattr(att, "ATTACHMENTS_DIR", tmp_path / "attachments")
|
|
monkeypatch.setattr(pf, "PROJECT_FILES_DIR", tmp_path / "project_files")
|
|
monkeypatch.setattr("devplacepy.config.CONTAINER_WORKSPACES_DIR", tmp_path / "workspaces")
|
|
|
|
attachments_dir = tmp_path / "attachments" / "ab" / "cd"
|
|
attachments_dir.mkdir(parents=True)
|
|
(attachments_dir / "orphan.png").write_bytes(b"a" * 10)
|
|
|
|
project_files_dir = tmp_path / "project_files" / "ab" / "cd"
|
|
project_files_dir.mkdir(parents=True)
|
|
(project_files_dir / "orphan.o").write_bytes(b"b" * 20)
|
|
|
|
(tmp_path / "workspaces").mkdir()
|
|
(tmp_path / "workspaces" / "orphan-project").mkdir()
|
|
|
|
cli.cmd_system_prune(argparse.Namespace(dry_run=True))
|
|
dry_output = capsys.readouterr().out
|
|
assert "DRY RUN" in dry_output
|
|
assert (attachments_dir / "orphan.png").exists()
|
|
assert (project_files_dir / "orphan.o").exists()
|
|
assert (tmp_path / "workspaces" / "orphan-project").exists()
|
|
|
|
cli.cmd_system_prune(argparse.Namespace(dry_run=False))
|
|
real_output = capsys.readouterr().out
|
|
assert "DRY RUN" not in real_output
|
|
assert not (attachments_dir / "orphan.png").exists()
|
|
assert not (project_files_dir / "orphan.o").exists()
|
|
assert not (tmp_path / "workspaces" / "orphan-project").exists()
|
|
|
|
|
|
def test_system_prune_registered_in_parser():
|
|
parser = cli.build_parser()
|
|
args = parser.parse_args(["system", "prune", "--dry-run"])
|
|
assert args.func is cli.cmd_system_prune
|
|
assert args.dry_run is True
|