devplacepy/nodes/extension/test/fetcher.test.mjs

import test from "node:test";
import assert from "node:assert/strict";
import { Fetcher } from "../src/Fetcher.js";
import { Limits } from "../src/Limits.js";
import { Policy } from "../src/Policy.js";
import { Usage } from "../src/Usage.js";
import { MAX_REDIRECTS, REFUSALS } from "../src/spec.js";

function memoryArea() {
  const store = {};
  return {
    async get(key) {
      return key in store ? { [key]: store[key] } : {};
    },
    async set(values) {
      Object.assign(store, values);
    },
  };
}

function build(fetchImpl, local = {}) {
  const limits = new Limits({ local });
  const usage = new Usage(memoryArea());
  const fetcher = new Fetcher({ policy: new Policy(), limits, usage, fetchImpl });
  return { fetcher, limits, usage };
}

const job = (url = "https://www.reddit.com/r/x.json", extra = {}) => ({
  job_id: "j1",
  method: "GET",
  url,
  timeout: 5,
  ...extra,
});

function response(body, init = {}) {
  return new Response(body, { status: 200, ...init });
}

test("success returns a job_result with base64 body and no donor credentials", async () => {
  const calls = [];
  const { fetcher, usage } = build(async (url, options) => {
    calls.push({ url, options });
    return response("hello", { headers: { "content-type": "text/plain", "set-cookie": "a=b" } });
  });
  const result = await fetcher.execute(
    job(undefined, { headers: { Cookie: "x=1", Authorization: "Bearer z", Accept: "text/html" } }),
  );
  assert.equal(result.type, "job_result");
  assert.equal(result.status, 200);
  assert.equal(Buffer.from(result.body_b64, "base64").toString(), "hello");
  assert.equal(result.truncated, false);
  assert.equal(result.headers["content-type"], "text/plain");
  assert.equal("set-cookie" in result.headers, false);
  assert.equal(calls[0].options.credentials, "omit");
  assert.equal(calls[0].options.redirect, "manual");
  assert.deepEqual(calls[0].options.headers, { accept: "text/html" });
  assert.equal(usage.snapshot().requests, 1);
  assert.equal(usage.snapshot().bytesDown, 5);
});

test("a disallowed url never reaches fetch", async () => {
  let called = 0;
  const { fetcher, limits } = build(async () => {
    called += 1;
    return response("x");
  });
  for (const url of ["https://evilreddit.com/", "http://reddit.com/", "https://reddit.com@evil.net/"]) {
    const result = await fetcher.execute(job(url));
    assert.equal(result.type, "job_error");
    assert.equal(result.code, REFUSALS.HOST_NOT_ALLOWED);
  }
  assert.equal(called, 0);
  assert.equal(limits.inflight, 0);
});

test("body over the cap is truncated and flagged", async () => {
  const big = "a".repeat(5000);
  const { fetcher, limits } = build(async () => response(big));
  limits.caps.max_body_bytes = 1000;
  const result = await fetcher.execute(job());
  assert.equal(result.type, "job_result");
  assert.equal(result.truncated, true);
  assert.equal(Buffer.from(result.body_b64, "base64").length, 1000);
});

test("the local bandwidth budget also bounds the body", async () => {
  const { fetcher, usage } = build(async () => response("b".repeat(3 * 1024 * 1024)), { bandwidthMb: 1 });
  const result = await fetcher.execute(job());
  assert.equal(result.truncated, true);
  assert.ok(Buffer.from(result.body_b64, "base64").length < 1024 * 1024);
  assert.ok(usage.usedBytes <= 1024 * 1024);
});

test("exhausted bandwidth refuses before fetching", async () => {
  let called = 0;
  const { fetcher, usage } = build(
    async () => {
      called += 1;
      return response("x");
    },
    { bandwidthMb: 1 },
  );
  await usage.record({ ok: true, up: 0, down: 1024 * 1024 });
  const result = await fetcher.execute(job());
  assert.equal(result.code, REFUSALS.BANDWIDTH_EXHAUSTED);
  assert.equal(called, 0);
});

test("concurrency cap refuses the extra job", async () => {
  let release;
  const gate = new Promise((resolve) => {
    release = resolve;
  });
  const { fetcher } = build(
    async () => {
      await gate;
      return response("x");
    },
    { maxConcurrent: 1 },
  );
  const first = fetcher.execute(job());
  const second = await fetcher.execute(job());
  assert.equal(second.code, REFUSALS.CONCURRENCY_EXCEEDED);
  release();
  assert.equal((await first).type, "job_result");
});

test("timeout aborts the request and reports timeout", async () => {
  const { fetcher, limits } = build(
    (url, options) =>
      new Promise((resolve, reject) => {
        options.signal.addEventListener("abort", () => reject(new DOMException("aborted", "AbortError")));
      }),
  );
  limits.caps.job_timeout_seconds = 1;
  const started = Date.now();
  const result = await fetcher.execute(job(undefined, { timeout: 1 }));
  assert.equal(result.code, REFUSALS.TIMEOUT);
  assert.ok(Date.now() - started < 3000);
  assert.equal(limits.inflight, 0);
});

test("a redirect leaving the allowlist is refused and not followed", async () => {
  const urls = [];
  const { fetcher } = build(async (url) => {
    urls.push(url);
    return response("", { status: 302, headers: { location: "https://evil.net/steal" } });
  });
  const result = await fetcher.execute(job());
  assert.equal(result.type, "job_error");
  assert.equal(result.code, REFUSALS.REDIRECT_NOT_ALLOWED);
  assert.equal(urls.length, 1);
});

test("an opaque redirect fails closed", async () => {
  const opaque = { type: "opaqueredirect", status: 0, headers: new Headers(), body: null };
  const { fetcher } = build(async () => opaque);
  const result = await fetcher.execute(job());
  assert.equal(result.code, REFUSALS.REDIRECT_NOT_ALLOWED);
});

test("a same-allowlist redirect is followed, with the policy checked on every hop", async () => {
  const urls = [];
  const { fetcher } = build(async (url) => {
    urls.push(url);
    if (urls.length === 1) {
      return response("", { status: 301, headers: { location: "https://old.reddit.com/r/x" } });
    }
    return response("final");
  });
  const result = await fetcher.execute(job());
  assert.equal(result.type, "job_result");
  assert.equal(Buffer.from(result.body_b64, "base64").toString(), "final");
  assert.deepEqual(urls, ["https://www.reddit.com/r/x.json", "https://old.reddit.com/r/x"]);
});

test("redirect loops are bounded", async () => {
  let count = 0;
  const { fetcher } = build(async () => {
    count += 1;
    return response("", { status: 302, headers: { location: "https://www.reddit.com/loop" } });
  });
  const result = await fetcher.execute(job());
  assert.equal(result.code, REFUSALS.REDIRECT_NOT_ALLOWED);
  assert.equal(count, MAX_REDIRECTS + 1);
});

test("a network failure is a loud job_error", async () => {
  const { fetcher, usage } = build(async () => {
    throw new TypeError("network down");
  });
  const result = await fetcher.execute(job());
  assert.equal(result.type, "job_error");
  assert.equal(result.code, "fetch_failed");
  assert.equal(usage.snapshot().failed, 1);
});