devplacepy/nodes/client/fetcher.go

package main

import (
	"context"
	"errors"
	"io"
	"net/http"
	"strings"
	"time"
)

var hopByHop = map[string]bool{
	"connection": true, "keep-alive": true, "proxy-authenticate": true, "proxy-authorization": true,
	"proxy-connection": true, "te": true, "trailer": true, "trailers": true,
	"transfer-encoding": true, "upgrade": true, "content-length": true, "host": true,
}

var identityHeaders = map[string]bool{
	"cookie": true, "cookie2": true, "authorization": true, "set-cookie": true, "set-cookie2": true,
}

type FetchResult struct {
	Status    int
	Headers   map[string]string
	Body      []byte
	Truncated bool
	Elapsed   time.Duration
}

type Fetcher struct {
	policy *Policy
	client *http.Client
}

func NewFetcher(policy *Policy) *Fetcher {
	transport := &http.Transport{
		Proxy:                  nil,
		DialContext:            policy.DialContext,
		ForceAttemptHTTP2:      true,
		TLSHandshakeTimeout:    10 * time.Second,
		ResponseHeaderTimeout:  20 * time.Second,
		MaxResponseHeaderBytes: 64 * 1024,
		MaxIdleConns:           4,
		MaxIdleConnsPerHost:    2,
		IdleConnTimeout:        30 * time.Second,
	}
	client := &http.Client{
		Transport: transport,
		Jar:       nil,
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			if len(via) > MaxRedirects {
				return refuse(RefuseRedirect, "too many redirects")
			}
			_, err := policy.CheckRedirect(req.URL.String(), req.Method)
			return err
		},
	}
	return &Fetcher{policy: policy, client: client}
}

func requestHeaders(source map[string]any) http.Header {
	header := http.Header{}
	count := 0
	for name, raw := range source {
		value, ok := raw.(string)
		lower := strings.ToLower(strings.TrimSpace(name))
		if !ok || lower == "" || hopByHop[lower] || identityHeaders[lower] || len(value) > MaxHeaderValueBytes {
			continue
		}
		if strings.ContainsAny(value, "\r\n") || strings.ContainsAny(lower, "\r\n: ") {
			continue
		}
		count++
		if count > MaxRequestHeaders {
			break
		}
		header.Set(lower, value)
	}
	return header
}

func responseHeaders(source http.Header) map[string]string {
	out := map[string]string{}
	for name, values := range source {
		lower := strings.ToLower(name)
		if hopByHop[lower] || identityHeaders[lower] || len(out) >= MaxResponseHeaders {
			continue
		}
		joined := strings.Join(values, ", ")
		if len(joined) > MaxHeaderValueBytes {
			joined = joined[:MaxHeaderValueBytes]
		}
		out[lower] = joined
	}
	return out
}

func ErrorCode(err error) string {
	var policyErr *PolicyError
	if errors.As(err, &policyErr) {
		return policyErr.Code
	}
	if errors.Is(err, context.DeadlineExceeded) {
		return RefuseTimeout
	}
	return FailFetch
}

func clampTimeout(seconds int) time.Duration {
	if seconds < 1 {
		seconds = DefaultJobTimeout
	}
	if seconds > MaxJobTimeout {
		seconds = MaxJobTimeout
	}
	return time.Duration(seconds) * time.Second
}

func (f *Fetcher) Do(ctx context.Context, frame Frame, maxBody int, budget int64) (*FetchResult, error) {
	target, err := f.policy.Check(frame.URL, frame.Method, "fetch")
	if err != nil {
		return nil, err
	}
	ctx, cancel := context.WithTimeout(ctx, clampTimeout(frame.Timeout))
	defer cancel()
	started := time.Now()
	request, err := http.NewRequestWithContext(ctx, target.Method, target.URL, nil)
	if err != nil {
		return nil, refuse(RefuseBadURL, "request cannot be built")
	}
	request.Header = requestHeaders(frame.Headers)
	response, err := f.client.Do(request)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	allowed := int64(maxBody)
	if budget < allowed {
		allowed = budget
	}
	body, err := io.ReadAll(io.LimitReader(response.Body, allowed+1))
	if err != nil {
		return nil, err
	}
	truncated := int64(len(body)) > allowed
	if truncated {
		body = body[:allowed]
	}
	return &FetchResult{
		Status:    response.StatusCode,
		Headers:   responseHeaders(response.Header),
		Body:      body,
		Truncated: truncated,
		Elapsed:   time.Since(started),
	}, nil
}