devplacepy/nodes/client/fetcher_test.go

package main

import (
	"context"
	"net"
	"net/http"
	"net/http/httptest"
	"net/netip"
	"strings"
	"sync"
	"testing"
	"time"
)

type fetchRig struct {
	policy *Policy
	f      *Fetcher
	mu     sync.Mutex
	dialed []string
}

func newFetchRig(t *testing.T, handler http.Handler) *fetchRig {
	t.Helper()
	server := httptest.NewServer(handler)
	t.Cleanup(server.Close)
	rig := &fetchRig{}
	rig.policy = testPolicy(publicAnswer)
	rig.policy.dial = func(ctx context.Context, network, address string) (net.Conn, error) {
		rig.mu.Lock()
		rig.dialed = append(rig.dialed, address)
		rig.mu.Unlock()
		var dialer net.Dialer
		return dialer.DialContext(ctx, "tcp", server.Listener.Addr().String())
	}
	rig.f = NewFetcher(rig.policy)
	return rig
}

func job(url string) Frame {
	return Frame{Type: "job", JobID: "j1", Method: "GET", URL: url, Timeout: 5}
}

func TestFetchSuccessStripsIdentityAndHopByHop(t *testing.T) {
	var seen http.Header
	rig := newFetchRig(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		seen = r.Header.Clone()
		w.Header().Set("Set-Cookie", "session=abc")
		w.Header().Set("X-Thing", "kept")
		w.Header().Set("Content-Type", "application/json")
		_, _ = w.Write([]byte(`{"ok":true}`))
	}))
	frame := job("http://www.reddit.com/r/go.json")
	frame.Headers = map[string]any{
		"Cookie": "a=b", "Authorization": "Bearer x", "Proxy-Authorization": "y", "Connection": "close",
		"Accept": "application/json", "X-Bad": "a\r\nb", "Host": "evil.com", "X-Number": 5,
	}
	result, err := rig.f.Do(context.Background(), frame, 1000, 1<<20)
	if err != nil {
		t.Fatal(err)
	}
	if result.Status != 200 || string(result.Body) != `{"ok":true}` || result.Truncated {
		t.Fatalf("result %+v", result)
	}
	for _, name := range []string{"Cookie", "Authorization", "Proxy-Authorization", "X-Bad", "X-Number"} {
		if seen.Get(name) != "" {
			t.Errorf("%s was forwarded", name)
		}
	}
	if seen.Get("Accept") != "application/json" {
		t.Error("allowed header dropped")
	}
	if _, ok := result.Headers["set-cookie"]; ok {
		t.Error("set-cookie reported")
	}
	if result.Headers["x-thing"] != "kept" {
		t.Errorf("headers %v", result.Headers)
	}
	if len(rig.dialed) != 1 || rig.dialed[0] != "93.184.216.34:80" {
		t.Errorf("dialed %v", rig.dialed)
	}
}

func TestFetchTruncatesAtBodyCapAndBudget(t *testing.T) {
	rig := newFetchRig(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		_, _ = w.Write([]byte(strings.Repeat("x", 5000)))
	}))
	result, err := rig.f.Do(context.Background(), job("http://reddit.com/a"), 1000, 1<<20)
	if err != nil || len(result.Body) != 1000 || !result.Truncated {
		t.Fatalf("cap: %v %+v", err, result)
	}
	result, err = rig.f.Do(context.Background(), job("http://reddit.com/a"), 1000, 300)
	if err != nil || len(result.Body) != 300 || !result.Truncated {
		t.Fatalf("budget: %v %+v", err, result)
	}
	result, err = rig.f.Do(context.Background(), job("http://reddit.com/a"), 10000, 1<<20)
	if err != nil || len(result.Body) != 5000 || result.Truncated {
		t.Fatalf("whole: %v %+v", err, result)
	}
}

func TestFetchRefusals(t *testing.T) {
	rig := newFetchRig(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		_, _ = w.Write([]byte("ok"))
	}))
	cases := []struct {
		name   string
		frame  Frame
		prep   func()
		expect string
	}{
		{"host", job("http://evilreddit.com/"), nil, RefuseHost},
		{"port", job("http://reddit.com:8080/"), nil, RefusePort},
		{"method", Frame{JobID: "j", Method: "POST", URL: "http://reddit.com/", Timeout: 5}, nil, RefuseMethod},
		{"literal", job("http://10.0.0.1/"), nil, RefuseHost},
		{"private resolve", job("http://reddit.com/"), func() {
			rig.policy.lookup = func(context.Context, string) ([]netip.Addr, error) {
				return []netip.Addr{netip.MustParseAddr("192.168.1.10")}, nil
			}
		}, RefuseAddress},
	}
	for _, item := range cases {
		if item.prep != nil {
			item.prep()
		}
		before := len(rig.dialed)
		_, err := rig.f.Do(context.Background(), item.frame, 1000, 1<<20)
		if got := ErrorCode(err); got != item.expect {
			t.Errorf("%s: got %q (%v)", item.name, got, err)
		}
		if len(rig.dialed) != before {
			t.Errorf("%s: a connection was attempted", item.name)
		}
	}
}

func TestFetchRedirects(t *testing.T) {
	var hits []string
	rig := newFetchRig(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		hits = append(hits, r.Host+r.URL.Path)
		switch r.URL.Path {
		case "/to-allowed":
			http.Redirect(w, r, "http://old.reddit.com/final", http.StatusFound)
		case "/to-evil":
			http.Redirect(w, r, "http://evil.example/steal", http.StatusFound)
		case "/to-private":
			http.Redirect(w, r, "http://127.0.0.1/admin", http.StatusFound)
		case "/to-port":
			http.Redirect(w, r, "http://reddit.com:8080/x", http.StatusFound)
		case "/loop":
			http.Redirect(w, r, "/loop", http.StatusFound)
		default:
			_, _ = w.Write([]byte("final"))
		}
	}))
	result, err := rig.f.Do(context.Background(), job("http://reddit.com/to-allowed"), 1000, 1<<20)
	if err != nil || string(result.Body) != "final" {
		t.Fatalf("allowed redirect: %v", err)
	}
	for _, path := range []string{"/to-evil", "/to-private", "/to-port", "/loop"} {
		_, err := rig.f.Do(context.Background(), job("http://reddit.com"+path), 1000, 1<<20)
		if ErrorCode(err) != RefuseRedirect {
			t.Errorf("%s: got %q (%v)", path, ErrorCode(err), err)
		}
	}
	for _, hit := range hits {
		if strings.HasPrefix(hit, "evil.example") || strings.HasPrefix(hit, "127.0.0.1") {
			t.Errorf("request reached %s", hit)
		}
	}
}

func TestFetchTimeout(t *testing.T) {
	rig := newFetchRig(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		select {
		case <-r.Context().Done():
		case <-time.After(3 * time.Second):
		}
	}))
	frame := job("http://reddit.com/slow")
	frame.Timeout = 1
	started := time.Now()
	_, err := rig.f.Do(context.Background(), frame, 1000, 1<<20)
	if ErrorCode(err) != RefuseTimeout {
		t.Fatalf("got %q (%v)", ErrorCode(err), err)
	}
	if time.Since(started) > 2500*time.Millisecond {
		t.Fatal("timeout was not enforced")
	}
}

func TestClampTimeout(t *testing.T) {
	cases := map[int]time.Duration{0: 25 * time.Second, -3: 25 * time.Second, 7: 7 * time.Second, 500: 60 * time.Second}
	for in, want := range cases {
		if got := clampTimeout(in); got != want {
			t.Errorf("%d: %v", in, got)
		}
	}
}