Zo doen wijdat in oost

2026-10-16 07:20:44 +02:00
parent 25d9cb1c7d
commit 6f80a820da
229 changed files with 23894 additions and 74 deletions
@@ -0,0 +1,198 @@
+package main
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+type fetchRig struct {
+	policy *Policy
+	f      *Fetcher
+	mu     sync.Mutex
+	dialed []string
+}
+
+func newFetchRig(t *testing.T, handler http.Handler) *fetchRig {
+	t.Helper()
+	server := httptest.NewServer(handler)
+	t.Cleanup(server.Close)
+	rig := &fetchRig{}
+	rig.policy = testPolicy(publicAnswer)
+	rig.policy.dial = func(ctx context.Context, network, address string) (net.Conn, error) {
+		rig.mu.Lock()
+		rig.dialed = append(rig.dialed, address)
+		rig.mu.Unlock()
+		var dialer net.Dialer
+		return dialer.DialContext(ctx, "tcp", server.Listener.Addr().String())
+	}
+	rig.f = NewFetcher(rig.policy)
+	return rig
+}
+
+func job(url string) Frame {
+	return Frame{Type: "job", JobID: "j1", Method: "GET", URL: url, Timeout: 5}
+}
+
+func TestFetchSuccessStripsIdentityAndHopByHop(t *testing.T) {
+	var seen http.Header
+	rig := newFetchRig(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		seen = r.Header.Clone()
+		w.Header().Set("Set-Cookie", "session=abc")
+		w.Header().Set("X-Thing", "kept")
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"ok":true}`))
+	}))
+	frame := job("http://www.reddit.com/r/go.json")
+	frame.Headers = map[string]any{
+		"Cookie": "a=b", "Authorization": "Bearer x", "Proxy-Authorization": "y", "Connection": "close",
+		"Accept": "application/json", "X-Bad": "a\r\nb", "Host": "evil.com", "X-Number": 5,
+	}
+	result, err := rig.f.Do(context.Background(), frame, 1000, 1<<20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.Status != 200 || string(result.Body) != `{"ok":true}` || result.Truncated {
+		t.Fatalf("result %+v", result)
+	}
+	for _, name := range []string{"Cookie", "Authorization", "Proxy-Authorization", "X-Bad", "X-Number"} {
+		if seen.Get(name) != "" {
+			t.Errorf("%s was forwarded", name)
+		}
+	}
+	if seen.Get("Accept") != "application/json" {
+		t.Error("allowed header dropped")
+	}
+	if _, ok := result.Headers["set-cookie"]; ok {
+		t.Error("set-cookie reported")
+	}
+	if result.Headers["x-thing"] != "kept" {
+		t.Errorf("headers %v", result.Headers)
+	}
+	if len(rig.dialed) != 1 || rig.dialed[0] != "93.184.216.34:80" {
+		t.Errorf("dialed %v", rig.dialed)
+	}
+}
+
+func TestFetchTruncatesAtBodyCapAndBudget(t *testing.T) {
+	rig := newFetchRig(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(strings.Repeat("x", 5000)))
+	}))
+	result, err := rig.f.Do(context.Background(), job("http://reddit.com/a"), 1000, 1<<20)
+	if err != nil || len(result.Body) != 1000 || !result.Truncated {
+		t.Fatalf("cap: %v %+v", err, result)
+	}
+	result, err = rig.f.Do(context.Background(), job("http://reddit.com/a"), 1000, 300)
+	if err != nil || len(result.Body) != 300 || !result.Truncated {
+		t.Fatalf("budget: %v %+v", err, result)
+	}
+	result, err = rig.f.Do(context.Background(), job("http://reddit.com/a"), 10000, 1<<20)
+	if err != nil || len(result.Body) != 5000 || result.Truncated {
+		t.Fatalf("whole: %v %+v", err, result)
+	}
+}
+
+func TestFetchRefusals(t *testing.T) {
+	rig := newFetchRig(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte("ok"))
+	}))
+	cases := []struct {
+		name   string
+		frame  Frame
+		prep   func()
+		expect string
+	}{
+		{"host", job("http://evilreddit.com/"), nil, RefuseHost},
+		{"port", job("http://reddit.com:8080/"), nil, RefusePort},
+		{"method", Frame{JobID: "j", Method: "POST", URL: "http://reddit.com/", Timeout: 5}, nil, RefuseMethod},
+		{"literal", job("http://10.0.0.1/"), nil, RefuseHost},
+		{"private resolve", job("http://reddit.com/"), func() {
+			rig.policy.lookup = func(context.Context, string) ([]netip.Addr, error) {
+				return []netip.Addr{netip.MustParseAddr("192.168.1.10")}, nil
+			}
+		}, RefuseAddress},
+	}
+	for _, item := range cases {
+		if item.prep != nil {
+			item.prep()
+		}
+		before := len(rig.dialed)
+		_, err := rig.f.Do(context.Background(), item.frame, 1000, 1<<20)
+		if got := ErrorCode(err); got != item.expect {
+			t.Errorf("%s: got %q (%v)", item.name, got, err)
+		}
+		if len(rig.dialed) != before {
+			t.Errorf("%s: a connection was attempted", item.name)
+		}
+	}
+}
+
+func TestFetchRedirects(t *testing.T) {
+	var hits []string
+	rig := newFetchRig(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits = append(hits, r.Host+r.URL.Path)
+		switch r.URL.Path {
+		case "/to-allowed":
+			http.Redirect(w, r, "http://old.reddit.com/final", http.StatusFound)
+		case "/to-evil":
+			http.Redirect(w, r, "http://evil.example/steal", http.StatusFound)
+		case "/to-private":
+			http.Redirect(w, r, "http://127.0.0.1/admin", http.StatusFound)
+		case "/to-port":
+			http.Redirect(w, r, "http://reddit.com:8080/x", http.StatusFound)
+		case "/loop":
+			http.Redirect(w, r, "/loop", http.StatusFound)
+		default:
+			_, _ = w.Write([]byte("final"))
+		}
+	}))
+	result, err := rig.f.Do(context.Background(), job("http://reddit.com/to-allowed"), 1000, 1<<20)
+	if err != nil || string(result.Body) != "final" {
+		t.Fatalf("allowed redirect: %v", err)
+	}
+	for _, path := range []string{"/to-evil", "/to-private", "/to-port", "/loop"} {
+		_, err := rig.f.Do(context.Background(), job("http://reddit.com"+path), 1000, 1<<20)
+		if ErrorCode(err) != RefuseRedirect {
+			t.Errorf("%s: got %q (%v)", path, ErrorCode(err), err)
+		}
+	}
+	for _, hit := range hits {
+		if strings.HasPrefix(hit, "evil.example") || strings.HasPrefix(hit, "127.0.0.1") {
+			t.Errorf("request reached %s", hit)
+		}
+	}
+}
+
+func TestFetchTimeout(t *testing.T) {
+	rig := newFetchRig(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		select {
+		case <-r.Context().Done():
+		case <-time.After(3 * time.Second):
+		}
+	}))
+	frame := job("http://reddit.com/slow")
+	frame.Timeout = 1
+	started := time.Now()
+	_, err := rig.f.Do(context.Background(), frame, 1000, 1<<20)
+	if ErrorCode(err) != RefuseTimeout {
+		t.Fatalf("got %q (%v)", ErrorCode(err), err)
+	}
+	if time.Since(started) > 2500*time.Millisecond {
+		t.Fatal("timeout was not enforced")
+	}
+}
+
+func TestClampTimeout(t *testing.T) {
+	cases := map[int]time.Duration{0: 25 * time.Second, -3: 25 * time.Second, 7: 7 * time.Second, 500: 60 * time.Second}
+	for in, want := range cases {
+		if got := clampTimeout(in); got != want {
+			t.Errorf("%d: %v", in, got)
+		}
+	}
+}