forked from retoor/devplacepy
Zo doen wijdat in oost
This commit is contained in:
@@ -0,0 +1,197 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { Fetcher } from "../src/Fetcher.js";
|
||||
import { Limits } from "../src/Limits.js";
|
||||
import { Policy } from "../src/Policy.js";
|
||||
import { Usage } from "../src/Usage.js";
|
||||
import { MAX_REDIRECTS, REFUSALS } from "../src/spec.js";
|
||||
|
||||
function memoryArea() {
|
||||
const store = {};
|
||||
return {
|
||||
async get(key) {
|
||||
return key in store ? { [key]: store[key] } : {};
|
||||
},
|
||||
async set(values) {
|
||||
Object.assign(store, values);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function build(fetchImpl, local = {}) {
|
||||
const limits = new Limits({ local });
|
||||
const usage = new Usage(memoryArea());
|
||||
const fetcher = new Fetcher({ policy: new Policy(), limits, usage, fetchImpl });
|
||||
return { fetcher, limits, usage };
|
||||
}
|
||||
|
||||
const job = (url = "https://www.reddit.com/r/x.json", extra = {}) => ({
|
||||
job_id: "j1",
|
||||
method: "GET",
|
||||
url,
|
||||
timeout: 5,
|
||||
...extra,
|
||||
});
|
||||
|
||||
function response(body, init = {}) {
|
||||
return new Response(body, { status: 200, ...init });
|
||||
}
|
||||
|
||||
test("success returns a job_result with base64 body and no donor credentials", async () => {
|
||||
const calls = [];
|
||||
const { fetcher, usage } = build(async (url, options) => {
|
||||
calls.push({ url, options });
|
||||
return response("hello", { headers: { "content-type": "text/plain", "set-cookie": "a=b" } });
|
||||
});
|
||||
const result = await fetcher.execute(
|
||||
job(undefined, { headers: { Cookie: "x=1", Authorization: "Bearer z", Accept: "text/html" } }),
|
||||
);
|
||||
assert.equal(result.type, "job_result");
|
||||
assert.equal(result.status, 200);
|
||||
assert.equal(Buffer.from(result.body_b64, "base64").toString(), "hello");
|
||||
assert.equal(result.truncated, false);
|
||||
assert.equal(result.headers["content-type"], "text/plain");
|
||||
assert.equal("set-cookie" in result.headers, false);
|
||||
assert.equal(calls[0].options.credentials, "omit");
|
||||
assert.equal(calls[0].options.redirect, "manual");
|
||||
assert.deepEqual(calls[0].options.headers, { accept: "text/html" });
|
||||
assert.equal(usage.snapshot().requests, 1);
|
||||
assert.equal(usage.snapshot().bytesDown, 5);
|
||||
});
|
||||
|
||||
test("a disallowed url never reaches fetch", async () => {
|
||||
let called = 0;
|
||||
const { fetcher, limits } = build(async () => {
|
||||
called += 1;
|
||||
return response("x");
|
||||
});
|
||||
for (const url of ["https://evilreddit.com/", "http://reddit.com/", "https://reddit.com@evil.net/"]) {
|
||||
const result = await fetcher.execute(job(url));
|
||||
assert.equal(result.type, "job_error");
|
||||
assert.equal(result.code, REFUSALS.HOST_NOT_ALLOWED);
|
||||
}
|
||||
assert.equal(called, 0);
|
||||
assert.equal(limits.inflight, 0);
|
||||
});
|
||||
|
||||
test("body over the cap is truncated and flagged", async () => {
|
||||
const big = "a".repeat(5000);
|
||||
const { fetcher, limits } = build(async () => response(big));
|
||||
limits.caps.max_body_bytes = 1000;
|
||||
const result = await fetcher.execute(job());
|
||||
assert.equal(result.type, "job_result");
|
||||
assert.equal(result.truncated, true);
|
||||
assert.equal(Buffer.from(result.body_b64, "base64").length, 1000);
|
||||
});
|
||||
|
||||
test("the local bandwidth budget also bounds the body", async () => {
|
||||
const { fetcher, usage } = build(async () => response("b".repeat(3 * 1024 * 1024)), { bandwidthMb: 1 });
|
||||
const result = await fetcher.execute(job());
|
||||
assert.equal(result.truncated, true);
|
||||
assert.ok(Buffer.from(result.body_b64, "base64").length < 1024 * 1024);
|
||||
assert.ok(usage.usedBytes <= 1024 * 1024);
|
||||
});
|
||||
|
||||
test("exhausted bandwidth refuses before fetching", async () => {
|
||||
let called = 0;
|
||||
const { fetcher, usage } = build(
|
||||
async () => {
|
||||
called += 1;
|
||||
return response("x");
|
||||
},
|
||||
{ bandwidthMb: 1 },
|
||||
);
|
||||
await usage.record({ ok: true, up: 0, down: 1024 * 1024 });
|
||||
const result = await fetcher.execute(job());
|
||||
assert.equal(result.code, REFUSALS.BANDWIDTH_EXHAUSTED);
|
||||
assert.equal(called, 0);
|
||||
});
|
||||
|
||||
test("concurrency cap refuses the extra job", async () => {
|
||||
let release;
|
||||
const gate = new Promise((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const { fetcher } = build(
|
||||
async () => {
|
||||
await gate;
|
||||
return response("x");
|
||||
},
|
||||
{ maxConcurrent: 1 },
|
||||
);
|
||||
const first = fetcher.execute(job());
|
||||
const second = await fetcher.execute(job());
|
||||
assert.equal(second.code, REFUSALS.CONCURRENCY_EXCEEDED);
|
||||
release();
|
||||
assert.equal((await first).type, "job_result");
|
||||
});
|
||||
|
||||
test("timeout aborts the request and reports timeout", async () => {
|
||||
const { fetcher, limits } = build(
|
||||
(url, options) =>
|
||||
new Promise((resolve, reject) => {
|
||||
options.signal.addEventListener("abort", () => reject(new DOMException("aborted", "AbortError")));
|
||||
}),
|
||||
);
|
||||
limits.caps.job_timeout_seconds = 1;
|
||||
const started = Date.now();
|
||||
const result = await fetcher.execute(job(undefined, { timeout: 1 }));
|
||||
assert.equal(result.code, REFUSALS.TIMEOUT);
|
||||
assert.ok(Date.now() - started < 3000);
|
||||
assert.equal(limits.inflight, 0);
|
||||
});
|
||||
|
||||
test("a redirect leaving the allowlist is refused and not followed", async () => {
|
||||
const urls = [];
|
||||
const { fetcher } = build(async (url) => {
|
||||
urls.push(url);
|
||||
return response("", { status: 302, headers: { location: "https://evil.net/steal" } });
|
||||
});
|
||||
const result = await fetcher.execute(job());
|
||||
assert.equal(result.type, "job_error");
|
||||
assert.equal(result.code, REFUSALS.REDIRECT_NOT_ALLOWED);
|
||||
assert.equal(urls.length, 1);
|
||||
});
|
||||
|
||||
test("an opaque redirect fails closed", async () => {
|
||||
const opaque = { type: "opaqueredirect", status: 0, headers: new Headers(), body: null };
|
||||
const { fetcher } = build(async () => opaque);
|
||||
const result = await fetcher.execute(job());
|
||||
assert.equal(result.code, REFUSALS.REDIRECT_NOT_ALLOWED);
|
||||
});
|
||||
|
||||
test("a same-allowlist redirect is followed, with the policy checked on every hop", async () => {
|
||||
const urls = [];
|
||||
const { fetcher } = build(async (url) => {
|
||||
urls.push(url);
|
||||
if (urls.length === 1) {
|
||||
return response("", { status: 301, headers: { location: "https://old.reddit.com/r/x" } });
|
||||
}
|
||||
return response("final");
|
||||
});
|
||||
const result = await fetcher.execute(job());
|
||||
assert.equal(result.type, "job_result");
|
||||
assert.equal(Buffer.from(result.body_b64, "base64").toString(), "final");
|
||||
assert.deepEqual(urls, ["https://www.reddit.com/r/x.json", "https://old.reddit.com/r/x"]);
|
||||
});
|
||||
|
||||
test("redirect loops are bounded", async () => {
|
||||
let count = 0;
|
||||
const { fetcher } = build(async () => {
|
||||
count += 1;
|
||||
return response("", { status: 302, headers: { location: "https://www.reddit.com/loop" } });
|
||||
});
|
||||
const result = await fetcher.execute(job());
|
||||
assert.equal(result.code, REFUSALS.REDIRECT_NOT_ALLOWED);
|
||||
assert.equal(count, MAX_REDIRECTS + 1);
|
||||
});
|
||||
|
||||
test("a network failure is a loud job_error", async () => {
|
||||
const { fetcher, usage } = build(async () => {
|
||||
throw new TypeError("network down");
|
||||
});
|
||||
const result = await fetcher.execute(job());
|
||||
assert.equal(result.type, "job_error");
|
||||
assert.equal(result.code, "fetch_failed");
|
||||
assert.equal(usage.snapshot().failed, 1);
|
||||
});
|
||||
Reference in New Issue
Block a user