@@ -0,0 +1,47 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import hashlib
|
||||
|
||||
import pytest
|
||||
|
||||
from devplacepy import config
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def artifacts(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(config, "NODE_CLIENTS_DIR", tmp_path)
|
||||
(tmp_path / "devplace-node-linux-amd64").write_bytes(b"binary-bytes")
|
||||
(tmp_path / "manifest.json").write_text('[{"version": "0.3.0"}]')
|
||||
(tmp_path / "bad name.txt").write_text("x")
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_client_version_lists_artifacts_with_checksums(client, artifacts):
|
||||
body = client.get("/nodes/client-version").json()
|
||||
assert body["version"] == "0.3.0" and body["protocol"] >= 1
|
||||
names = [item["name"] for item in body["artifacts"]]
|
||||
assert names == ["devplace-node-linux-amd64"]
|
||||
assert body["artifacts"][0]["sha256"] == hashlib.sha256(b"binary-bytes").hexdigest()
|
||||
|
||||
|
||||
def test_download_serves_the_file_with_its_checksum(client, artifacts):
|
||||
response = client.get("/nodes/download/devplace-node-linux-amd64")
|
||||
assert response.status_code == 200 and response.content == b"binary-bytes"
|
||||
assert response.headers["x-content-sha256"] == hashlib.sha256(b"binary-bytes").hexdigest()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["manifest.json", "missing", "..%2Fsecret", "bad%20name.txt", ".hidden"])
|
||||
def test_download_refuses_everything_else(client, artifacts, name):
|
||||
assert client.get(f"/nodes/download/{name}").status_code in (404, 422)
|
||||
|
||||
|
||||
def test_download_refuses_a_symlink(client, artifacts):
|
||||
outside = artifacts.parent / "outside.bin"
|
||||
outside.write_bytes(b"secret")
|
||||
(artifacts / "link").symlink_to(outside)
|
||||
assert client.get("/nodes/download/link").status_code == 404
|
||||
|
||||
|
||||
def test_empty_directory_is_an_empty_list(client, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(config, "NODE_CLIENTS_DIR", tmp_path / "absent")
|
||||
assert client.get("/nodes/client-version").json()["artifacts"] == []
|
||||
@@ -0,0 +1,187 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from starlette.websockets import WebSocketDisconnect
|
||||
|
||||
from devplacepy.database import get_table
|
||||
from devplacepy.database import nodes as store
|
||||
from devplacepy.services.manager import service_manager
|
||||
from devplacepy.services.nodes.broker import JobBroker
|
||||
from devplacepy.services.nodes.policy import DestinationPolicy
|
||||
from devplacepy.services.nodes.spec import CLOSE_PROTOCOL, SPEC
|
||||
from tests.api.nodes.conftest import make_member
|
||||
from tests.api.nodes.ws import hello, ticket_for
|
||||
|
||||
URL = "https://www.reddit.com/r/python.json"
|
||||
BROKER = JobBroker()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def relaxed(client):
|
||||
service = service_manager.get_service("nodes")
|
||||
original = service.scheduler.destination_ready
|
||||
service.scheduler.destination_ready = lambda target: True
|
||||
get_table("node_requests").delete()
|
||||
yield service
|
||||
service.scheduler.destination_ready = original
|
||||
get_table("node_requests").delete()
|
||||
|
||||
|
||||
def enqueue(caller="caller-1", kind="user", url=URL):
|
||||
target = DestinationPolicy().check(url)
|
||||
return BROKER.enqueue(kind, caller, target, "any")
|
||||
|
||||
|
||||
def connect(client, member):
|
||||
device_uid, _, ticket = ticket_for(client, member)
|
||||
return device_uid, client.websocket_connect(f"/nodes/ws?ticket={ticket}")
|
||||
|
||||
|
||||
def next_job(ws, timeout=5.0):
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
frame = json.loads(ws.receive_text())
|
||||
if frame["type"] == "job":
|
||||
return frame
|
||||
if frame["type"] == "ping":
|
||||
ws.send_text(json.dumps({"type": "pong"}))
|
||||
raise AssertionError("no job arrived")
|
||||
|
||||
|
||||
def wait_for(uid, status, timeout=5.0):
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
row = store.get_request(uid)
|
||||
if row["status"] == status:
|
||||
return row
|
||||
time.sleep(0.05)
|
||||
raise AssertionError(f"request stayed {store.get_request(uid)['status']}")
|
||||
|
||||
|
||||
def test_a_queued_request_reaches_the_node_and_the_result_comes_back(client, relaxed, member):
|
||||
device_uid, connection = connect(client, member)
|
||||
with connection as ws:
|
||||
ws.send_text(hello(modes=["fetch"]))
|
||||
assert json.loads(ws.receive_text())["type"] == "welcome"
|
||||
uid = enqueue()
|
||||
job = next_job(ws)
|
||||
assert job["job_id"] == uid and job["url"] == URL and job["method"] == "GET"
|
||||
assert job["timeout"] == SPEC.limits.job_timeout_seconds
|
||||
ws.send_text(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "job_result",
|
||||
"job_id": uid,
|
||||
"status": 200,
|
||||
"headers": {"Content-Type": "application/json"},
|
||||
"body_b64": base64.b64encode(b'{"ok": true}').decode(),
|
||||
}
|
||||
)
|
||||
)
|
||||
wait_for(uid, "done")
|
||||
result = BROKER.read(uid)
|
||||
assert result.http_status == 200 and result.body == b'{"ok": true}'
|
||||
assert result.headers == {"content-type": "application/json"}
|
||||
today = store.usage_today(device_uid)
|
||||
assert today["requests_ok"] == 1 and today["bytes_down"] == 12
|
||||
assert store.get_donor(member["uid"])["requests_ok"] == 1
|
||||
|
||||
|
||||
def test_a_node_that_speaks_no_known_transport_gets_nothing(client, relaxed, member):
|
||||
_, connection = connect(client, member)
|
||||
with connection as ws:
|
||||
ws.send_text(hello(modes=["any"]))
|
||||
ws.receive_text()
|
||||
uid = enqueue()
|
||||
time.sleep(0.7)
|
||||
assert store.get_request(uid)["status"] == "queued"
|
||||
|
||||
|
||||
def test_a_donor_never_receives_its_own_request(client, relaxed, member):
|
||||
_, connection = connect(client, member)
|
||||
with connection as ws:
|
||||
ws.send_text(hello(modes=["fetch"]))
|
||||
ws.receive_text()
|
||||
uid = enqueue(caller=member["uid"])
|
||||
time.sleep(0.7)
|
||||
assert store.get_request(uid)["status"] == "queued"
|
||||
|
||||
|
||||
def test_a_donor_opt_out_of_the_platform_is_respected_live(client, relaxed, member):
|
||||
store.save_node_settings(member["uid"], {"allowed_platforms": []})
|
||||
_, connection = connect(client, member)
|
||||
with connection as ws:
|
||||
ws.send_text(hello(modes=["fetch"]))
|
||||
assert json.loads(ws.receive_text())["settings"]["allowed_platforms"] == []
|
||||
uid = enqueue()
|
||||
time.sleep(0.7)
|
||||
assert store.get_request(uid)["status"] == "queued"
|
||||
|
||||
|
||||
def test_a_job_error_marks_the_request_failed(client, relaxed, member):
|
||||
_, connection = connect(client, member)
|
||||
with connection as ws:
|
||||
ws.send_text(hello(modes=["fetch"]))
|
||||
ws.receive_text()
|
||||
uid = enqueue()
|
||||
job = next_job(ws)
|
||||
ws.send_text(json.dumps({"type": "job_error", "job_id": job["job_id"], "code": "timeout"}))
|
||||
row = wait_for(uid, "failed")
|
||||
assert row["error"] == "timeout"
|
||||
|
||||
|
||||
def test_disconnecting_with_a_job_in_flight_fails_it(client, relaxed, member):
|
||||
_, connection = connect(client, member)
|
||||
with connection as ws:
|
||||
ws.send_text(hello(modes=["fetch"]))
|
||||
ws.receive_text()
|
||||
uid = enqueue()
|
||||
next_job(ws)
|
||||
row = wait_for(uid, "failed")
|
||||
assert row["error"] == "node_disconnected"
|
||||
|
||||
|
||||
def test_a_forged_result_is_an_integrity_failure_and_a_flag(client, relaxed, member):
|
||||
_, connection = connect(client, member)
|
||||
with connection as ws:
|
||||
ws.send_text(hello(modes=["fetch"]))
|
||||
ws.receive_text()
|
||||
uid = enqueue()
|
||||
next_job(ws)
|
||||
ws.send_text(
|
||||
json.dumps({"type": "job_result", "job_id": uid, "status": 200, "body_b64": "***"})
|
||||
)
|
||||
row = wait_for(uid, "failed")
|
||||
assert row["error"] == "bad_base64"
|
||||
assert any(flag["kind"] == "integrity" for flag in store.list_flags())
|
||||
|
||||
|
||||
def test_answering_a_job_that_was_never_assigned_closes_the_socket(client, relaxed, member):
|
||||
_, connection = connect(client, member)
|
||||
with connection as ws:
|
||||
ws.send_text(hello(modes=["fetch"]))
|
||||
ws.receive_text()
|
||||
ws.send_text(
|
||||
json.dumps({"type": "job_result", "job_id": "made-up", "status": 200, "body_b64": ""})
|
||||
)
|
||||
with pytest.raises(WebSocketDisconnect) as info:
|
||||
ws.receive_text()
|
||||
assert info.value.code == CLOSE_PROTOCOL
|
||||
|
||||
|
||||
def test_two_donors_split_two_requests(client, relaxed):
|
||||
first, second = make_member(), make_member()
|
||||
_, one = connect(client, first)
|
||||
_, two = connect(client, second)
|
||||
with one as ws_one, two as ws_two:
|
||||
for ws in (ws_one, ws_two):
|
||||
ws.send_text(hello(modes=["fetch"]))
|
||||
ws.receive_text()
|
||||
a = enqueue(caller="x")
|
||||
b = enqueue(caller="y", url="https://old.reddit.com/r/rust.json")
|
||||
jobs = {next_job(ws_one)["job_id"], next_job(ws_two)["job_id"]}
|
||||
assert jobs == {a, b}
|
||||
@@ -0,0 +1,84 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from devplacepy.database import set_consent
|
||||
from devplacepy.database.nodes import record_job, save_node_settings
|
||||
from tests.api.nodes.conftest import make_member
|
||||
|
||||
|
||||
def test_guests_get_401(client):
|
||||
assert client.get("/nodes/me").status_code == 401
|
||||
assert client.post("/nodes/settings", json={"enabled": False}).status_code == 401
|
||||
|
||||
|
||||
def test_me_reports_status_and_never_caches(client, member):
|
||||
response = client.get("/nodes/me", headers=member["headers"])
|
||||
assert response.status_code == 200
|
||||
assert response.headers["cache-control"] == "no-store"
|
||||
body = response.json()
|
||||
assert body["consent"] is True and body["can_donate"] is True
|
||||
assert body["settings"]["enabled"] is True
|
||||
assert body["devices"] == [] and body["max_devices"] > 0
|
||||
assert body["donated"]["lifetime_requests"] == 0
|
||||
assert "reddit" in body["platforms"]
|
||||
|
||||
|
||||
def test_me_explains_why_a_member_cannot_donate(client):
|
||||
who = make_member(consent=False, enabled=False)
|
||||
body = client.get("/nodes/me", headers=who["headers"]).json()
|
||||
assert body["can_donate"] is False
|
||||
assert body["reason"] and body["message"]
|
||||
|
||||
|
||||
def test_settings_are_clamped_and_unsent_fields_keep_their_value(client, member):
|
||||
response = client.post(
|
||||
"/nodes/settings",
|
||||
json={"bandwidth_mb_day": 100, "active_from": "08:00", "active_to": "22:00", "timezone": "Europe/Amsterdam"},
|
||||
headers=member["headers"],
|
||||
)
|
||||
assert response.status_code == 200
|
||||
settings = response.json()["settings"]
|
||||
assert settings["bandwidth_mb_day"] == 100 and settings["active_from"] == "08:00"
|
||||
assert settings["enabled"] is True
|
||||
again = client.post("/nodes/settings", json={"max_concurrent": 1}, headers=member["headers"]).json()
|
||||
assert again["settings"]["bandwidth_mb_day"] == 100 and again["settings"]["max_concurrent"] == 1
|
||||
too_big = client.post("/nodes/settings", json={"bandwidth_mb_day": 10**9}, headers=member["headers"])
|
||||
assert too_big.status_code in (400, 422)
|
||||
|
||||
|
||||
def test_switching_on_needs_the_consent(client):
|
||||
who = make_member(consent=False, enabled=False)
|
||||
refused = client.post("/nodes/settings", json={"enabled": True}, headers=who["headers"])
|
||||
assert refused.status_code == 403
|
||||
assert refused.json()["error"]["reason"] == "no_consent"
|
||||
set_consent("user", who["uid"], "bandwidth_sharing", True)
|
||||
assert client.post("/nodes/settings", json={"enabled": True}, headers=who["headers"]).status_code == 200
|
||||
|
||||
|
||||
def test_the_assistant_can_switch_off_but_never_on(client):
|
||||
who = make_member(enabled=False)
|
||||
headers = {**who["headers"], "X-Devii-Agent": "1"}
|
||||
refused = client.post("/nodes/settings", json={"enabled": True}, headers=headers)
|
||||
assert refused.status_code == 403
|
||||
save_node_settings(who["uid"], {"enabled": True})
|
||||
off = client.post("/nodes/settings", json={"enabled": False}, headers=headers)
|
||||
assert off.status_code == 200 and off.json()["settings"]["enabled"] is False
|
||||
|
||||
|
||||
def test_stats_are_aggregate_only(client, member):
|
||||
device = client.post("/nodes/devices", json={"label": "x"}, headers=member["headers"]).json()["device"]
|
||||
record_job(device["uid"], member["uid"], ok=True, bytes_up=10, bytes_down=90)
|
||||
body = client.get("/nodes/stats").json()
|
||||
assert set(body) == {"requests", "bytes", "donors"}
|
||||
assert body["requests"] >= 1 and body["bytes"] >= 100 and body["donors"] >= 1
|
||||
assert member["username"] not in str(body)
|
||||
|
||||
|
||||
def test_leaderboard_lists_only_members_who_opted_in(client):
|
||||
hidden = make_member()
|
||||
shown = make_member()
|
||||
for who, public in ((hidden, False), (shown, True)):
|
||||
save_node_settings(who["uid"], {"public_display": public})
|
||||
device = client.post("/nodes/devices", json={"label": "x"}, headers=who["headers"]).json()["device"]
|
||||
record_job(device["uid"], who["uid"], ok=True, bytes_up=1, bytes_down=5)
|
||||
names = [item["username"] for item in client.get("/nodes/leaderboard").json()["donors"]]
|
||||
assert shown["username"] in names and hidden["username"] not in names
|
||||
@@ -0,0 +1,65 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import json
|
||||
|
||||
from devplacepy.database import get_setting, set_setting
|
||||
from devplacepy.services.nodes import social as social_module
|
||||
|
||||
HN = json.dumps({"hits": [{"objectID": "9", "title": "A story", "points": 4}]}).encode()
|
||||
|
||||
|
||||
def test_guests_get_401(client):
|
||||
assert client.get("/nodes/social/search?q=x").status_code == 401
|
||||
assert client.post("/nodes/internal/fetch", json={"url": "https://www.reddit.com/"}).status_code == 401
|
||||
|
||||
|
||||
def test_query_is_required(client, member):
|
||||
assert client.get("/nodes/social/search", headers=member["headers"]).status_code == 400
|
||||
|
||||
|
||||
def test_search_returns_hits_and_reasons(client, member, monkeypatch):
|
||||
async def getter(url):
|
||||
return HN
|
||||
|
||||
monkeypatch.setattr(social_module, "direct_get", getter)
|
||||
body = client.get(
|
||||
"/nodes/social/search?q=sqlite&platforms=hackernews,reddit", headers=member["headers"]
|
||||
).json()
|
||||
assert body["hits"][0]["platform"] == "hackernews"
|
||||
assert body["unavailable"] == {"reddit": "nodes_not_used"}
|
||||
assert body["platforms"] == ["hackernews", "reddit"]
|
||||
|
||||
|
||||
def test_search_has_a_daily_quota(client, member, monkeypatch):
|
||||
async def getter(url):
|
||||
return HN
|
||||
|
||||
monkeypatch.setattr(social_module, "direct_get", getter)
|
||||
previous = get_setting("nodes_daily_social_user", "300")
|
||||
set_setting("nodes_daily_social_user", "2")
|
||||
try:
|
||||
codes = [
|
||||
client.get("/nodes/social/search?q=a&platforms=hackernews", headers=member["headers"]).status_code
|
||||
for _ in range(3)
|
||||
]
|
||||
finally:
|
||||
set_setting("nodes_daily_social_user", previous)
|
||||
assert codes == [200, 200, 429]
|
||||
|
||||
|
||||
def test_internal_fetch_refuses_hosts_off_the_allowlist(client, member):
|
||||
response = client.post(
|
||||
"/nodes/internal/fetch", json={"url": "https://example.com/"}, headers=member["headers"]
|
||||
)
|
||||
assert response.status_code == 403
|
||||
assert response.json()["error"]["reason"] == "host_not_allowed"
|
||||
|
||||
|
||||
def test_internal_fetch_without_donors_is_503(client, member):
|
||||
response = client.post(
|
||||
"/nodes/internal/fetch",
|
||||
json={"url": "https://www.reddit.com/r/python.json", "cache": False},
|
||||
headers=member["headers"],
|
||||
)
|
||||
assert response.status_code == 503
|
||||
assert response.json()["error"]["reason"] in ("no_nodes", "kill_switch")
|
||||
Reference in New Issue
Block a user