Zo doen wijdat in oost

2026-10-16 07:20:44 +02:00
parent 25d9cb1c7d
commit 6f80a820da
229 changed files with 23894 additions and 74 deletions
@@ -0,0 +1,156 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+)
+
+var hopByHop = map[string]bool{
+	"connection": true, "keep-alive": true, "proxy-authenticate": true, "proxy-authorization": true,
+	"proxy-connection": true, "te": true, "trailer": true, "trailers": true,
+	"transfer-encoding": true, "upgrade": true, "content-length": true, "host": true,
+}
+
+var identityHeaders = map[string]bool{
+	"cookie": true, "cookie2": true, "authorization": true, "set-cookie": true, "set-cookie2": true,
+}
+
+type FetchResult struct {
+	Status    int
+	Headers   map[string]string
+	Body      []byte
+	Truncated bool
+	Elapsed   time.Duration
+}
+
+type Fetcher struct {
+	policy *Policy
+	client *http.Client
+}
+
+func NewFetcher(policy *Policy) *Fetcher {
+	transport := &http.Transport{
+		Proxy:                  nil,
+		DialContext:            policy.DialContext,
+		ForceAttemptHTTP2:      true,
+		TLSHandshakeTimeout:    10 * time.Second,
+		ResponseHeaderTimeout:  20 * time.Second,
+		MaxResponseHeaderBytes: 64 * 1024,
+		MaxIdleConns:           4,
+		MaxIdleConnsPerHost:    2,
+		IdleConnTimeout:        30 * time.Second,
+	}
+	client := &http.Client{
+		Transport: transport,
+		Jar:       nil,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if len(via) > MaxRedirects {
+				return refuse(RefuseRedirect, "too many redirects")
+			}
+			_, err := policy.CheckRedirect(req.URL.String(), req.Method)
+			return err
+		},
+	}
+	return &Fetcher{policy: policy, client: client}
+}
+
+func requestHeaders(source map[string]any) http.Header {
+	header := http.Header{}
+	count := 0
+	for name, raw := range source {
+		value, ok := raw.(string)
+		lower := strings.ToLower(strings.TrimSpace(name))
+		if !ok || lower == "" || hopByHop[lower] || identityHeaders[lower] || len(value) > MaxHeaderValueBytes {
+			continue
+		}
+		if strings.ContainsAny(value, "\r\n") || strings.ContainsAny(lower, "\r\n: ") {
+			continue
+		}
+		count++
+		if count > MaxRequestHeaders {
+			break
+		}
+		header.Set(lower, value)
+	}
+	return header
+}
+
+func responseHeaders(source http.Header) map[string]string {
+	out := map[string]string{}
+	for name, values := range source {
+		lower := strings.ToLower(name)
+		if hopByHop[lower] || identityHeaders[lower] || len(out) >= MaxResponseHeaders {
+			continue
+		}
+		joined := strings.Join(values, ", ")
+		if len(joined) > MaxHeaderValueBytes {
+			joined = joined[:MaxHeaderValueBytes]
+		}
+		out[lower] = joined
+	}
+	return out
+}
+
+func ErrorCode(err error) string {
+	var policyErr *PolicyError
+	if errors.As(err, &policyErr) {
+		return policyErr.Code
+	}
+	if errors.Is(err, context.DeadlineExceeded) {
+		return RefuseTimeout
+	}
+	return FailFetch
+}
+
+func clampTimeout(seconds int) time.Duration {
+	if seconds < 1 {
+		seconds = DefaultJobTimeout
+	}
+	if seconds > MaxJobTimeout {
+		seconds = MaxJobTimeout
+	}
+	return time.Duration(seconds) * time.Second
+}
+
+func (f *Fetcher) Do(ctx context.Context, frame Frame, maxBody int, budget int64) (*FetchResult, error) {
+	target, err := f.policy.Check(frame.URL, frame.Method, "fetch")
+	if err != nil {
+		return nil, err
+	}
+	ctx, cancel := context.WithTimeout(ctx, clampTimeout(frame.Timeout))
+	defer cancel()
+	started := time.Now()
+	request, err := http.NewRequestWithContext(ctx, target.Method, target.URL, nil)
+	if err != nil {
+		return nil, refuse(RefuseBadURL, "request cannot be built")
+	}
+	request.Header = requestHeaders(frame.Headers)
+	response, err := f.client.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+	allowed := int64(maxBody)
+	if budget < allowed {
+		allowed = budget
+	}
+	body, err := io.ReadAll(io.LimitReader(response.Body, allowed+1))
+	if err != nil {
+		return nil, err
+	}
+	truncated := int64(len(body)) > allowed
+	if truncated {
+		body = body[:allowed]
+	}
+	return &FetchResult{
+		Status:    response.StatusCode,
+		Headers:   responseHeaders(response.Header),
+		Body:      body,
+		Truncated: truncated,
+		Elapsed:   time.Since(started),
+	}, nil
+}