import test from "node:test"; import assert from "node:assert/strict"; import { Fetcher } from "../src/Fetcher.js"; import { Limits } from "../src/Limits.js"; import { Policy } from "../src/Policy.js"; import { Usage } from "../src/Usage.js"; import { MAX_REDIRECTS, REFUSALS } from "../src/spec.js"; function memoryArea() { const store = {}; return { async get(key) { return key in store ? { [key]: store[key] } : {}; }, async set(values) { Object.assign(store, values); }, }; } function build(fetchImpl, local = {}) { const limits = new Limits({ local }); const usage = new Usage(memoryArea()); const fetcher = new Fetcher({ policy: new Policy(), limits, usage, fetchImpl }); return { fetcher, limits, usage }; } const job = (url = "https://www.reddit.com/r/x.json", extra = {}) => ({ job_id: "j1", method: "GET", url, timeout: 5, ...extra, }); function response(body, init = {}) { return new Response(body, { status: 200, ...init }); } test("success returns a job_result with base64 body and no donor credentials", async () => { const calls = []; const { fetcher, usage } = build(async (url, options) => { calls.push({ url, options }); return response("hello", { headers: { "content-type": "text/plain", "set-cookie": "a=b" } }); }); const result = await fetcher.execute( job(undefined, { headers: { Cookie: "x=1", Authorization: "Bearer z", Accept: "text/html" } }), ); assert.equal(result.type, "job_result"); assert.equal(result.status, 200); assert.equal(Buffer.from(result.body_b64, "base64").toString(), "hello"); assert.equal(result.truncated, false); assert.equal(result.headers["content-type"], "text/plain"); assert.equal("set-cookie" in result.headers, false); assert.equal(calls[0].options.credentials, "omit"); assert.equal(calls[0].options.redirect, "manual"); assert.deepEqual(calls[0].options.headers, { accept: "text/html" }); assert.equal(usage.snapshot().requests, 1); assert.equal(usage.snapshot().bytesDown, 5); }); test("a disallowed url never reaches fetch", async () => { let called = 0; const { fetcher, limits } = build(async () => { called += 1; return response("x"); }); for (const url of ["https://evilreddit.com/", "http://reddit.com/", "https://reddit.com@evil.net/"]) { const result = await fetcher.execute(job(url)); assert.equal(result.type, "job_error"); assert.equal(result.code, REFUSALS.HOST_NOT_ALLOWED); } assert.equal(called, 0); assert.equal(limits.inflight, 0); }); test("body over the cap is truncated and flagged", async () => { const big = "a".repeat(5000); const { fetcher, limits } = build(async () => response(big)); limits.caps.max_body_bytes = 1000; const result = await fetcher.execute(job()); assert.equal(result.type, "job_result"); assert.equal(result.truncated, true); assert.equal(Buffer.from(result.body_b64, "base64").length, 1000); }); test("the local bandwidth budget also bounds the body", async () => { const { fetcher, usage } = build(async () => response("b".repeat(3 * 1024 * 1024)), { bandwidthMb: 1 }); const result = await fetcher.execute(job()); assert.equal(result.truncated, true); assert.ok(Buffer.from(result.body_b64, "base64").length < 1024 * 1024); assert.ok(usage.usedBytes <= 1024 * 1024); }); test("exhausted bandwidth refuses before fetching", async () => { let called = 0; const { fetcher, usage } = build( async () => { called += 1; return response("x"); }, { bandwidthMb: 1 }, ); await usage.record({ ok: true, up: 0, down: 1024 * 1024 }); const result = await fetcher.execute(job()); assert.equal(result.code, REFUSALS.BANDWIDTH_EXHAUSTED); assert.equal(called, 0); }); test("concurrency cap refuses the extra job", async () => { let release; const gate = new Promise((resolve) => { release = resolve; }); const { fetcher } = build( async () => { await gate; return response("x"); }, { maxConcurrent: 1 }, ); const first = fetcher.execute(job()); const second = await fetcher.execute(job()); assert.equal(second.code, REFUSALS.CONCURRENCY_EXCEEDED); release(); assert.equal((await first).type, "job_result"); }); test("timeout aborts the request and reports timeout", async () => { const { fetcher, limits } = build( (url, options) => new Promise((resolve, reject) => { options.signal.addEventListener("abort", () => reject(new DOMException("aborted", "AbortError"))); }), ); limits.caps.job_timeout_seconds = 1; const started = Date.now(); const result = await fetcher.execute(job(undefined, { timeout: 1 })); assert.equal(result.code, REFUSALS.TIMEOUT); assert.ok(Date.now() - started < 3000); assert.equal(limits.inflight, 0); }); test("a redirect leaving the allowlist is refused and not followed", async () => { const urls = []; const { fetcher } = build(async (url) => { urls.push(url); return response("", { status: 302, headers: { location: "https://evil.net/steal" } }); }); const result = await fetcher.execute(job()); assert.equal(result.type, "job_error"); assert.equal(result.code, REFUSALS.REDIRECT_NOT_ALLOWED); assert.equal(urls.length, 1); }); test("an opaque redirect fails closed", async () => { const opaque = { type: "opaqueredirect", status: 0, headers: new Headers(), body: null }; const { fetcher } = build(async () => opaque); const result = await fetcher.execute(job()); assert.equal(result.code, REFUSALS.REDIRECT_NOT_ALLOWED); }); test("a same-allowlist redirect is followed, with the policy checked on every hop", async () => { const urls = []; const { fetcher } = build(async (url) => { urls.push(url); if (urls.length === 1) { return response("", { status: 301, headers: { location: "https://old.reddit.com/r/x" } }); } return response("final"); }); const result = await fetcher.execute(job()); assert.equal(result.type, "job_result"); assert.equal(Buffer.from(result.body_b64, "base64").toString(), "final"); assert.deepEqual(urls, ["https://www.reddit.com/r/x.json", "https://old.reddit.com/r/x"]); }); test("redirect loops are bounded", async () => { let count = 0; const { fetcher } = build(async () => { count += 1; return response("", { status: 302, headers: { location: "https://www.reddit.com/loop" } }); }); const result = await fetcher.execute(job()); assert.equal(result.code, REFUSALS.REDIRECT_NOT_ALLOWED); assert.equal(count, MAX_REDIRECTS + 1); }); test("a network failure is a loud job_error", async () => { const { fetcher, usage } = build(async () => { throw new TypeError("network down"); }); const result = await fetcher.execute(job()); assert.equal(result.type, "job_error"); assert.equal(result.code, "fetch_failed"); assert.equal(usage.snapshot().failed, 1); });