blindxfish/devplacepy
1
0
Fork 0
forked from retoor/devplacepy
Code Pull Requests Activity

Zo doen wijdat in oost

Browse Source
This commit is contained in:
retoor
2026-10-16 07:20:44 +02:00
parent 25d9cb1c7d
commit 6f80a820da
229 changed files with 23894 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nodes/client/README.md
+62
View File
@@ -0,0 +1,62 @@
# devplace-node
Native donor client for the DevPlace community node network. One static Go binary, standard library only.
## What you share
When running, and only inside the limits you configured:
- Your internet connection and your IP address, used to fetch public pages from the destinations on the allowlist (today: `reddit.com`, `redd.it` and their subdomains) over ports 80 and 443, GET only.
- The fetched public page, which DevPlace receives and may cache for a short time.
- Byte counters (up and down) so your daily cap and your lifetime totals are correct.
## What is never shared
- Your cookies, logins, accounts and browser. The client has no cookie jar and drops `Cookie`, `Authorization` and `Proxy-Authorization` from every request. It uses no browser profile and reads no files besides its own config.
- Any access to your local network: addresses that are private, loopback, link-local, CGNAT or multicast are refused after local DNS resolution, and the connection goes to the address that was checked.
- Any destination outside the baked-in allowlist. A list sent by the server can only be equal or narrower. Redirects that leave the allowlist are refused.
- Anything on ports other than 80 and 443.
The device secret (`dpn_...`) is stored in the config file with mode 0600 and is never logged or printed.
## Use
```
devplace-node login --server https://devplace.net --secret dpn_...
devplace-node status
devplace-node run
devplace-node version
```
Create the device secret on your DevPlace profile after consenting to bandwidth sharing. `login` asks the server to accept the secret before storing it (`--no-verify` skips that). The secret can also come from `DEVPLACE_NODE_SECRET`. Every command takes `--config PATH`.
`run` is quiet: it logs start, disconnects, and one totals line per day. Add `-v` for detail. Exit code 3 means the server ended this device for good (revoked, suspended, kill switch, no consent, not adult, replaced by another instance of the same device); fix the cause and start it again.
## Config file
Stored at `<user config dir>/devplace-node/config.json`, with `usage.json` (today's byte count) beside it.
```json
{
"server": "https://devplace.net",
"secret": "dpn_...",
"bandwidth_mb_day": 500,
"max_concurrent": 2,
"max_tunnels": 2,
"active_from": "",
"active_to": "",
"timezone": ""
}
```
`active_from` and `active_to` are `HH:MM` in `timezone` (empty means the system timezone); a window may wrap midnight. These limits are enforced locally and the server can only make them stricter.
## Build and package
`./build.sh [outdir]` cross-compiles linux/amd64, linux/arm64, darwin/arm64, darwin/amd64 and windows/amd64 with `CGO_ENABLED=0 -trimpath`, and writes `manifest.json` (name, os, arch, sha256, size, version) beside the binaries. `VERSION=1.2.3 ./build.sh` overrides the version.
`packaging/` holds a systemd user unit (`loginctl enable-linger $USER` keeps it running without a login), a launchd plist, a Windows scheduled-task script and a container compose file.
## Tests
`go vet ./... && go test ./...`
nodes/client/build.sh
Executable
+54
View File
@@ -0,0 +1,54 @@
#!/bin/sh
set -eu
here=$(cd "$(dirname "$0")" && pwd)
out=${1:-./dist}
version=${VERSION:-$(git -C "$here" describe --tags --always --dirty 2>/dev/null || echo dev)}
targets="linux/amd64 linux/arm64 darwin/arm64 darwin/amd64 windows/amd64"
mkdir -p "$out"
out=$(cd "$out" && pwd)
manifest="$out/manifest.json"
rm -f "$manifest"
hash_of() {
if command -v sha256sum >/dev/null 2>&1; then
sha256sum "$1" | cut -d' ' -f1
else
shasum -a 256 "$1" | cut -d' ' -f1
fi
}
size_of() {
wc -c <"$1" | tr -d ' '
}
printf '[\n' >"$manifest"
first=1
for target in $targets; do
goos=${target%/*}
goarch=${target#*/}
name="devplace-node-$goos-$goarch"
if [ "$goos" = "windows" ]; then
name="$name.exe"
fi
(cd "$here" && CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -trimpath -ldflags "-s -w -X main.Version=$version" -o "$out/$name" .)
if [ "$first" -eq 0 ]; then
printf ',\n' >>"$manifest"
fi
first=0
printf ' {"name": "%s", "os": "%s", "arch": "%s", "sha256": "%s", "size": %s, "version": "%s"}' \
"$name" "$goos" "$goarch" "$(hash_of "$out/$name")" "$(size_of "$out/$name")" "$version" >>"$manifest"
done
printf '\n]\n' >>"$manifest"
for target in $targets; do
goos=${target%/*}
goarch=${target#*/}
name="devplace-node-$goos-$goarch"
if [ "$goos" = "windows" ]; then
name="$name.exe"
fi
echo "$out/$name"
done
echo "$manifest"
nodes/client/config.go
+177
View File
@@ -0,0 +1,177 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"net/url"
"os"
"path/filepath"
"regexp"
"strings"
"time"
)
const (
DefaultBandwidthMB = 500
DefaultMaxConcurrent = 2
DefaultMaxTunnels = 2
ConfigDirName = "devplace-node"
ConfigFileName = "config.json"
UsageFileName = "usage.json"
maxConfigBytes = 64 * 1024
)
var hoursPattern = regexp.MustCompile(`^([01]\d|2[0-3]):[0-5]\d$`)
type Config struct {
Server string `json:"server"`
Secret string `json:"secret"`
BandwidthMBDay int `json:"bandwidth_mb_day"`
MaxConcurrent int `json:"max_concurrent"`
MaxTunnels int `json:"max_tunnels"`
ActiveFrom string `json:"active_from"`
ActiveTo string `json:"active_to"`
Timezone string `json:"timezone"`
}
func DefaultConfig() Config {
return Config{
BandwidthMBDay: DefaultBandwidthMB,
MaxConcurrent: DefaultMaxConcurrent,
MaxTunnels: DefaultMaxTunnels,
}
}
func DefaultConfigPath() (string, error) {
base, err := os.UserConfigDir()
if err != nil {
return "", fmt.Errorf("cannot find the user config directory: %w", err)
}
return filepath.Join(base, ConfigDirName, ConfigFileName), nil
}
func (c Config) Validate() error {
parsed, err := url.Parse(c.Server)
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {
return errors.New("server must be an http or https url such as https://devplace.net")
}
if !strings.HasPrefix(c.Secret, SecretPrefix) || len(c.Secret) < len(SecretPrefix)+8 {
return errors.New("secret must be a device secret starting with " + SecretPrefix)
}
if c.BandwidthMBDay < 1 || c.BandwidthMBDay > MaxDonorMBPerDay {
return fmt.Errorf("bandwidth_mb_day must be between 1 and %d", MaxDonorMBPerDay)
}
if c.MaxConcurrent < 1 || c.MaxConcurrent > MaxConcurrentCeiling {
return fmt.Errorf("max_concurrent must be between 1 and %d", MaxConcurrentCeiling)
}
if c.MaxTunnels < 1 || c.MaxTunnels > MaxTunnelsCeiling {
return fmt.Errorf("max_tunnels must be between 1 and %d", MaxTunnelsCeiling)
}
for _, value := range []string{c.ActiveFrom, c.ActiveTo} {
if value != "" && !hoursPattern.MatchString(value) {
return errors.New("active_from and active_to must be empty or HH:MM")
}
}
if (c.ActiveFrom == "") != (c.ActiveTo == "") {
return errors.New("active_from and active_to must be set together")
}
if _, err := c.Location(); err != nil {
return err
}
return nil
}
func (c Config) Location() (*time.Location, error) {
if c.Timezone == "" {
return time.Local, nil
}
location, err := time.LoadLocation(c.Timezone)
if err != nil {
return nil, fmt.Errorf("timezone %q is not known", c.Timezone)
}
return location, nil
}
func (c Config) BaseURL() string {
return strings.TrimRight(c.Server, "/")
}
func LoadConfig(path string) (Config, error) {
file, err := os.Open(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return Config{}, fmt.Errorf("no config at %s, run: devplace-node login --server URL --secret dpn_...", path)
}
return Config{}, err
}
defer file.Close()
config := DefaultConfig()
decoder := json.NewDecoder(&limitedReader{r: file, left: maxConfigBytes})
decoder.DisallowUnknownFields()
if err := decoder.Decode(&config); err != nil {
return Config{}, fmt.Errorf("config %s is invalid: %w", path, err)
}
if err := config.Validate(); err != nil {
return Config{}, fmt.Errorf("config %s is invalid: %w", path, err)
}
return config, nil
}
type limitedReader struct {
r *os.File
left int
}
func (l *limitedReader) Read(p []byte) (int, error) {
if l.left <= 0 {
return 0, errors.New("file is too large")
}
if len(p) > l.left {
p = p[:l.left]
}
n, err := l.r.Read(p)
l.left -= n
return n, err
}
func SaveConfig(path string, config Config) error {
if err := config.Validate(); err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
data, err := json.MarshalIndent(config, "", " ")
if err != nil {
return err
}
return writeFileAtomic(path, append(data, '\n'))
}
func writeFileAtomic(path string, data []byte) error {
temp, err := os.CreateTemp(filepath.Dir(path), ".tmp-*")
if err != nil {
return err
}
name := temp.Name()
if err := temp.Chmod(0o600); err != nil {
temp.Close()
os.Remove(name)
return err
}
if _, err := temp.Write(data); err != nil {
temp.Close()
os.Remove(name)
return err
}
if err := temp.Close(); err != nil {
os.Remove(name)
return err
}
if err := os.Rename(name, path); err != nil {
os.Remove(name)
return err
}
return nil
}
nodes/client/config_test.go
+116
View File
@@ -0,0 +1,116 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
)
func validConfig() Config {
config := DefaultConfig()
config.Server = "https://devplace.net"
config.Secret = "dpn_abcdefghijklmnop"
return config
}
func TestConfigRoundTripShapeAndMode(t *testing.T) {
path := filepath.Join(t.TempDir(), "sub", "config.json")
if err := SaveConfig(path, validConfig()); err != nil {
t.Fatal(err)
}
raw, _ := os.ReadFile(path)
keys := map[string]any{}
if err := json.Unmarshal(raw, &keys); err != nil {
t.Fatal(err)
}
for _, key := range []string{"server", "secret", "bandwidth_mb_day", "max_concurrent", "max_tunnels", "active_from", "active_to", "timezone"} {
if _, ok := keys[key]; !ok {
t.Errorf("missing key %s", key)
}
}
if len(keys) != 8 {
t.Errorf("unexpected keys %v", keys)
}
if runtime.GOOS != "windows" {
info, _ := os.Stat(path)
if info.Mode().Perm() != 0o600 {
t.Errorf("mode %v", info.Mode().Perm())
}
}
loaded, err := LoadConfig(path)
if err != nil || loaded != validConfig() {
t.Fatalf("loaded %+v %v", loaded, err)
}
}
func TestConfigValidationTable(t *testing.T) {
cases := map[string]func(*Config){
"server scheme": func(c *Config) { c.Server = "ftp://x" },
"server empty": func(c *Config) { c.Server = "" },
"secret prefix": func(c *Config) { c.Secret = "abc_1234567890" },
"secret short": func(c *Config) { c.Secret = "dpn_" },
"bandwidth low": func(c *Config) { c.BandwidthMBDay = 0 },
"bandwidth high": func(c *Config) { c.BandwidthMBDay = MaxDonorMBPerDay + 1 },
"concurrency": func(c *Config) { c.MaxConcurrent = 99 },
"tunnels": func(c *Config) { c.MaxTunnels = 0 },
"hours format": func(c *Config) { c.ActiveFrom, c.ActiveTo = "9:00", "17:00" },
"hours lone": func(c *Config) { c.ActiveFrom = "09:00" },
"timezone": func(c *Config) { c.Timezone = "Mars/Base" },
}
for name, mutate := range cases {
config := validConfig()
mutate(&config)
if config.Validate() == nil {
t.Errorf("%s accepted", name)
}
}
good := validConfig()
good.ActiveFrom, good.ActiveTo, good.Timezone = "22:00", "06:00", "Europe/Amsterdam"
if err := good.Validate(); err != nil {
t.Errorf("good config: %v", err)
}
}
func TestLoadRejectsUnknownFieldsAndMissingFile(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "config.json")
if _, err := LoadConfig(path); err == nil || !strings.Contains(err.Error(), "login") {
t.Errorf("missing file: %v", err)
}
_ = os.WriteFile(path, []byte(`{"server":"https://x.y","secret":"dpn_abcdefghijk","surprise":1}`), 0o600)
if _, err := LoadConfig(path); err == nil {
t.Error("unknown field accepted")
}
}
func TestCommandsVersionAndUsage(t *testing.T) {
var out, errOut strings.Builder
if code := execute([]string{"version"}, &out, &errOut); code != exitOK || !strings.Contains(out.String(), Version) {
t.Fatalf("version: %d %q", code, out.String())
}
if code := execute(nil, &out, &errOut); code != exitUsage {
t.Fatalf("no args: %d", code)
}
if code := execute([]string{"bogus"}, &out, &errOut); code != exitUsage {
t.Fatalf("bogus: %d", code)
}
}
func TestLoginStoresWithoutVerifyAndNeverEchoesSecret(t *testing.T) {
path := filepath.Join(t.TempDir(), "config.json")
var out, errOut strings.Builder
args := []string{"--config", path, "--server", "https://devplace.net", "--secret", "dpn_abcdefghijklmnop", "--no-verify"}
if code := execute(append([]string{"login"}, args...), &out, &errOut); code != exitOK {
t.Fatalf("login: %d %s", code, errOut.String())
}
if strings.Contains(out.String()+errOut.String(), "dpn_abcdefghijklmnop") {
t.Fatal("secret was printed")
}
out.Reset()
if code := execute([]string{"login", "--config", path, "--server", "nope", "--secret", "x"}, &out, &errOut); code != exitUsage {
t.Fatalf("bad login accepted: %d", code)
}
}
nodes/client/fetcher.go
+156
View File
@@ -0,0 +1,156 @@
package main
import (
"context"
"errors"
"io"
"net/http"
"strings"
"time"
)
var hopByHop = map[string]bool{
"connection": true, "keep-alive": true, "proxy-authenticate": true, "proxy-authorization": true,
"proxy-connection": true, "te": true, "trailer": true, "trailers": true,
"transfer-encoding": true, "upgrade": true, "content-length": true, "host": true,
}
var identityHeaders = map[string]bool{
"cookie": true, "cookie2": true, "authorization": true, "set-cookie": true, "set-cookie2": true,
}
type FetchResult struct {
Status int
Headers map[string]string
Body []byte
Truncated bool
Elapsed time.Duration
}
type Fetcher struct {
policy *Policy
client *http.Client
}
func NewFetcher(policy *Policy) *Fetcher {
transport := &http.Transport{
Proxy: nil,
DialContext: policy.DialContext,
ForceAttemptHTTP2: true,
TLSHandshakeTimeout: 10 * time.Second,
ResponseHeaderTimeout: 20 * time.Second,
MaxResponseHeaderBytes: 64 * 1024,
MaxIdleConns: 4,
MaxIdleConnsPerHost: 2,
IdleConnTimeout: 30 * time.Second,
}
client := &http.Client{
Transport: transport,
Jar: nil,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
if len(via) > MaxRedirects {
return refuse(RefuseRedirect, "too many redirects")
}
_, err := policy.CheckRedirect(req.URL.String(), req.Method)
return err
},
}
return &Fetcher{policy: policy, client: client}
}
func requestHeaders(source map[string]any) http.Header {
header := http.Header{}
count := 0
for name, raw := range source {
value, ok := raw.(string)
lower := strings.ToLower(strings.TrimSpace(name))
if !ok || lower == "" || hopByHop[lower] || identityHeaders[lower] || len(value) > MaxHeaderValueBytes {
continue
}
if strings.ContainsAny(value, "\r\n") || strings.ContainsAny(lower, "\r\n: ") {
continue
}
count++
if count > MaxRequestHeaders {
break
}
header.Set(lower, value)
}
return header
}
func responseHeaders(source http.Header) map[string]string {
out := map[string]string{}
for name, values := range source {
lower := strings.ToLower(name)
if hopByHop[lower] || identityHeaders[lower] || len(out) >= MaxResponseHeaders {
continue
}
joined := strings.Join(values, ", ")
if len(joined) > MaxHeaderValueBytes {
joined = joined[:MaxHeaderValueBytes]
}
out[lower] = joined
}
return out
}
func ErrorCode(err error) string {
var policyErr *PolicyError
if errors.As(err, &policyErr) {
return policyErr.Code
}
if errors.Is(err, context.DeadlineExceeded) {
return RefuseTimeout
}
return FailFetch
}
func clampTimeout(seconds int) time.Duration {
if seconds < 1 {
seconds = DefaultJobTimeout
}
if seconds > MaxJobTimeout {
seconds = MaxJobTimeout
}
return time.Duration(seconds) * time.Second
}
func (f *Fetcher) Do(ctx context.Context, frame Frame, maxBody int, budget int64) (*FetchResult, error) {
target, err := f.policy.Check(frame.URL, frame.Method, "fetch")
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(ctx, clampTimeout(frame.Timeout))
defer cancel()
started := time.Now()
request, err := http.NewRequestWithContext(ctx, target.Method, target.URL, nil)
if err != nil {
return nil, refuse(RefuseBadURL, "request cannot be built")
}
request.Header = requestHeaders(frame.Headers)
response, err := f.client.Do(request)
if err != nil {
return nil, err
}
defer response.Body.Close()
allowed := int64(maxBody)
if budget < allowed {
allowed = budget
}
body, err := io.ReadAll(io.LimitReader(response.Body, allowed+1))
if err != nil {
return nil, err
}
truncated := int64(len(body)) > allowed
if truncated {
body = body[:allowed]
}
return &FetchResult{
Status: response.StatusCode,
Headers: responseHeaders(response.Header),
Body: body,
Truncated: truncated,
Elapsed: time.Since(started),
}, nil
}
nodes/client/fetcher_test.go
+198
View File
@@ -0,0 +1,198 @@
package main
import (
"context"
"net"
"net/http"
"net/http/httptest"
"net/netip"
"strings"
"sync"
"testing"
"time"
)
type fetchRig struct {
policy *Policy
f *Fetcher
mu sync.Mutex
dialed []string
}
func newFetchRig(t *testing.T, handler http.Handler) *fetchRig {
t.Helper()
server := httptest.NewServer(handler)
t.Cleanup(server.Close)
rig := &fetchRig{}
rig.policy = testPolicy(publicAnswer)
rig.policy.dial = func(ctx context.Context, network, address string) (net.Conn, error) {
rig.mu.Lock()
rig.dialed = append(rig.dialed, address)
rig.mu.Unlock()
var dialer net.Dialer
return dialer.DialContext(ctx, "tcp", server.Listener.Addr().String())
}
rig.f = NewFetcher(rig.policy)
return rig
}
func job(url string) Frame {
return Frame{Type: "job", JobID: "j1", Method: "GET", URL: url, Timeout: 5}
}
func TestFetchSuccessStripsIdentityAndHopByHop(t *testing.T) {
var seen http.Header
rig := newFetchRig(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
seen = r.Header.Clone()
w.Header().Set("Set-Cookie", "session=abc")
w.Header().Set("X-Thing", "kept")
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"ok":true}`))
}))
frame := job("http://www.reddit.com/r/go.json")
frame.Headers = map[string]any{
"Cookie": "a=b", "Authorization": "Bearer x", "Proxy-Authorization": "y", "Connection": "close",
"Accept": "application/json", "X-Bad": "a\r\nb", "Host": "evil.com", "X-Number": 5,
}
result, err := rig.f.Do(context.Background(), frame, 1000, 1<<20)
if err != nil {
t.Fatal(err)
}
if result.Status != 200 || string(result.Body) != `{"ok":true}` || result.Truncated {
t.Fatalf("result %+v", result)
}
for _, name := range []string{"Cookie", "Authorization", "Proxy-Authorization", "X-Bad", "X-Number"} {
if seen.Get(name) != "" {
t.Errorf("%s was forwarded", name)
}
}
if seen.Get("Accept") != "application/json" {
t.Error("allowed header dropped")
}
if _, ok := result.Headers["set-cookie"]; ok {
t.Error("set-cookie reported")
}
if result.Headers["x-thing"] != "kept" {
t.Errorf("headers %v", result.Headers)
}
if len(rig.dialed) != 1 || rig.dialed[0] != "93.184.216.34:80" {
t.Errorf("dialed %v", rig.dialed)
}
}
func TestFetchTruncatesAtBodyCapAndBudget(t *testing.T) {
rig := newFetchRig(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(strings.Repeat("x", 5000)))
}))
result, err := rig.f.Do(context.Background(), job("http://reddit.com/a"), 1000, 1<<20)
if err != nil || len(result.Body) != 1000 || !result.Truncated {
t.Fatalf("cap: %v %+v", err, result)
}
result, err = rig.f.Do(context.Background(), job("http://reddit.com/a"), 1000, 300)
if err != nil || len(result.Body) != 300 || !result.Truncated {
t.Fatalf("budget: %v %+v", err, result)
}
result, err = rig.f.Do(context.Background(), job("http://reddit.com/a"), 10000, 1<<20)
if err != nil || len(result.Body) != 5000 || result.Truncated {
t.Fatalf("whole: %v %+v", err, result)
}
}
func TestFetchRefusals(t *testing.T) {
rig := newFetchRig(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("ok"))
}))
cases := []struct {
name string
frame Frame
prep func()
expect string
}{
{"host", job("http://evilreddit.com/"), nil, RefuseHost},
{"port", job("http://reddit.com:8080/"), nil, RefusePort},
{"method", Frame{JobID: "j", Method: "POST", URL: "http://reddit.com/", Timeout: 5}, nil, RefuseMethod},
{"literal", job("http://10.0.0.1/"), nil, RefuseHost},
{"private resolve", job("http://reddit.com/"), func() {
rig.policy.lookup = func(context.Context, string) ([]netip.Addr, error) {
return []netip.Addr{netip.MustParseAddr("192.168.1.10")}, nil
}
}, RefuseAddress},
}
for _, item := range cases {
if item.prep != nil {
item.prep()
}
before := len(rig.dialed)
_, err := rig.f.Do(context.Background(), item.frame, 1000, 1<<20)
if got := ErrorCode(err); got != item.expect {
t.Errorf("%s: got %q (%v)", item.name, got, err)
}
if len(rig.dialed) != before {
t.Errorf("%s: a connection was attempted", item.name)
}
}
}
func TestFetchRedirects(t *testing.T) {
var hits []string
rig := newFetchRig(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits = append(hits, r.Host+r.URL.Path)
switch r.URL.Path {
case "/to-allowed":
http.Redirect(w, r, "http://old.reddit.com/final", http.StatusFound)
case "/to-evil":
http.Redirect(w, r, "http://evil.example/steal", http.StatusFound)
case "/to-private":
http.Redirect(w, r, "http://127.0.0.1/admin", http.StatusFound)
case "/to-port":
http.Redirect(w, r, "http://reddit.com:8080/x", http.StatusFound)
case "/loop":
http.Redirect(w, r, "/loop", http.StatusFound)
default:
_, _ = w.Write([]byte("final"))
}
}))
result, err := rig.f.Do(context.Background(), job("http://reddit.com/to-allowed"), 1000, 1<<20)
if err != nil || string(result.Body) != "final" {
t.Fatalf("allowed redirect: %v", err)
}
for _, path := range []string{"/to-evil", "/to-private", "/to-port", "/loop"} {
_, err := rig.f.Do(context.Background(), job("http://reddit.com"+path), 1000, 1<<20)
if ErrorCode(err) != RefuseRedirect {
t.Errorf("%s: got %q (%v)", path, ErrorCode(err), err)
}
}
for _, hit := range hits {
if strings.HasPrefix(hit, "evil.example") || strings.HasPrefix(hit, "127.0.0.1") {
t.Errorf("request reached %s", hit)
}
}
}
func TestFetchTimeout(t *testing.T) {
rig := newFetchRig(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
select {
case <-r.Context().Done():
case <-time.After(3 * time.Second):
}
}))
frame := job("http://reddit.com/slow")
frame.Timeout = 1
started := time.Now()
_, err := rig.f.Do(context.Background(), frame, 1000, 1<<20)
if ErrorCode(err) != RefuseTimeout {
t.Fatalf("got %q (%v)", ErrorCode(err), err)
}
if time.Since(started) > 2500*time.Millisecond {
t.Fatal("timeout was not enforced")
}
}
func TestClampTimeout(t *testing.T) {
cases := map[int]time.Duration{0: 25 * time.Second, -3: 25 * time.Second, 7: 7 * time.Second, 500: 60 * time.Second}
for in, want := range cases {
if got := clampTimeout(in); got != want {
t.Errorf("%d: %v", in, got)
}
}
}
nodes/client/go.mod
+3
View File
@@ -0,0 +1,3 @@
module devplace-node
go 1.22
nodes/client/limits.go
+238
View File
@@ -0,0 +1,238 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"sync"
"time"
)
const (
mebibyte = 1024 * 1024
persistInterval = 5 * time.Second
)
type usageState struct {
Day string `json:"day"`
Bytes int64 `json:"bytes"`
}
type Limiter struct {
mu sync.Mutex
cfg Config
loc *time.Location
now func() time.Time
statePath string
serverLimit ServerLimits
serverSet ServerSettings
hasServer bool
day string
used int64
active int
lastPersist time.Time
}
func NewLimiter(cfg Config, statePath string, now func() time.Time) (*Limiter, error) {
loc, err := cfg.Location()
if err != nil {
return nil, err
}
limiter := &Limiter{cfg: cfg, loc: loc, now: now, statePath: statePath}
limiter.load()
return limiter, nil
}
func (l *Limiter) load() {
if l.statePath == "" {
return
}
data, err := os.ReadFile(l.statePath)
if err != nil || len(data) > 4096 {
return
}
var state usageState
if json.Unmarshal(data, &state) != nil || state.Bytes < 0 {
return
}
l.day, l.used = state.Day, state.Bytes
}
func (l *Limiter) persist(force bool) {
if l.statePath == "" {
return
}
moment := l.now()
if !force && moment.Sub(l.lastPersist) < persistInterval {
return
}
l.lastPersist = moment
data, err := json.Marshal(usageState{Day: l.day, Bytes: l.used})
if err != nil {
return
}
if err := os.MkdirAll(filepath.Dir(l.statePath), 0o700); err != nil {
return
}
_ = writeFileAtomic(l.statePath, data)
}
func (l *Limiter) Flush() {
l.mu.Lock()
defer l.mu.Unlock()
l.roll()
l.persist(true)
}
func (l *Limiter) ApplyServer(limits *ServerLimits, settings *ServerSettings) {
l.mu.Lock()
defer l.mu.Unlock()
if limits != nil {
l.serverLimit = *limits
}
if settings != nil {
l.serverSet = *settings
}
l.hasServer = limits != nil || settings != nil
}
func (l *Limiter) roll() {
today := l.now().In(l.loc).Format("2006-01-02")
if l.day != today {
l.day = today
l.used = 0
}
}
func minPositive(values ...int) int {
best := 0
for _, value := range values {
if value > 0 && (best == 0 || value < best) {
best = value
}
}
return best
}
func (l *Limiter) capBytes() int64 {
mb := minPositive(l.cfg.BandwidthMBDay, l.serverSet.BandwidthMBDay, l.serverLimit.MaxDonorMBPerDay)
return int64(mb) * mebibyte
}
func (l *Limiter) concurrencyCap() int {
return minPositive(l.cfg.MaxConcurrent, l.serverSet.MaxConcurrent, l.serverLimit.MaxConcurrentPerNode)
}
func minutesOf(text string) int {
var hours, minutes int
if len(text) != 5 {
return -1
}
hours = int(text[0]-'0')*10 + int(text[1]-'0')
minutes = int(text[3]-'0')*10 + int(text[4]-'0')
return hours*60 + minutes
}
func InWindow(from, to string, moment time.Time) bool {
if from == "" || to == "" || from == to {
return true
}
start, end := minutesOf(from), minutesOf(to)
if start < 0 || end < 0 {
return true
}
now := moment.Hour()*60 + moment.Minute()
if start < end {
return now >= start && now < end
}
return now >= start || now < end
}
func (l *Limiter) hoursCode() string {
moment := l.now()
if !InWindow(l.cfg.ActiveFrom, l.cfg.ActiveTo, moment.In(l.loc)) {
return RefuseHours
}
if l.serverSet.ActiveFrom != "" {
zone, err := time.LoadLocation(l.serverSet.Timezone)
if err != nil {
zone = time.UTC
}
if !InWindow(l.serverSet.ActiveFrom, l.serverSet.ActiveTo, moment.In(zone)) {
return RefuseHours
}
}
return ""
}
func (l *Limiter) Acquire() string {
l.mu.Lock()
defer l.mu.Unlock()
l.roll()
if code := l.hoursCode(); code != "" {
return code
}
if l.used >= l.capBytes() {
return RefuseBandwidth
}
if l.active >= l.concurrencyCap() {
return RefuseConcurrency
}
l.active++
return ""
}
func (l *Limiter) Release() {
l.mu.Lock()
defer l.mu.Unlock()
if l.active > 0 {
l.active--
}
}
func (l *Limiter) Remaining() int64 {
l.mu.Lock()
defer l.mu.Unlock()
l.roll()
left := l.capBytes() - l.used
if left < 0 {
return 0
}
return left
}
func (l *Limiter) AddBytes(n int64) {
if n <= 0 {
return
}
l.mu.Lock()
defer l.mu.Unlock()
l.roll()
l.used += n
l.persist(false)
}
func (l *Limiter) Usage() (day string, used, capacity int64) {
l.mu.Lock()
defer l.mu.Unlock()
l.roll()
return l.day, l.used, l.capBytes()
}
func ReadUsage(path string, loc *time.Location, now time.Time) int64 {
data, err := os.ReadFile(path)
if err != nil || len(data) > 4096 {
return 0
}
var state usageState
if json.Unmarshal(data, &state) != nil || state.Day != now.In(loc).Format("2006-01-02") {
return 0
}
return state.Bytes
}
func (l *Limiter) ConcurrencyCap() int {
l.mu.Lock()
defer l.mu.Unlock()
return l.concurrencyCap()
}
nodes/client/limits_test.go
+162
View File
@@ -0,0 +1,162 @@
package main
import (
"path/filepath"
"testing"
"time"
)
type clock struct{ now time.Time }
func (c *clock) Now() time.Time { return c.now }
func testLimiter(t *testing.T, cfg Config, at string) (*Limiter, *clock) {
t.Helper()
moment, err := time.Parse(time.RFC3339, at)
if err != nil {
t.Fatal(err)
}
cfg.Timezone = "UTC"
c := &clock{now: moment}
limiter, err := NewLimiter(cfg, "", c.Now)
if err != nil {
t.Fatal(err)
}
return limiter, c
}
func TestInWindowTable(t *testing.T) {
at := func(text string) time.Time {
moment, _ := time.Parse("15:04", text)
return moment
}
cases := []struct {
from, to, now string
want bool
}{
{"", "", "12:00", true},
{"09:00", "17:00", "09:00", true},
{"09:00", "17:00", "16:59", true},
{"09:00", "17:00", "17:00", false},
{"09:00", "17:00", "08:59", false},
{"22:00", "06:00", "23:30", true},
{"22:00", "06:00", "05:59", true},
{"22:00", "06:00", "06:00", false},
{"22:00", "06:00", "12:00", false},
{"10:00", "10:00", "03:00", true},
}
for _, item := range cases {
if got := InWindow(item.from, item.to, at(item.now)); got != item.want {
t.Errorf("%s-%s at %s: got %v", item.from, item.to, item.now, got)
}
}
}
func TestAcquireConcurrencyCapAndServerCannotWiden(t *testing.T) {
cfg := DefaultConfig()
cfg.MaxConcurrent = 2
limiter, _ := testLimiter(t, cfg, "2026-10-15T12:00:00Z")
limiter.ApplyServer(&ServerLimits{MaxConcurrentPerNode: 8}, &ServerSettings{MaxConcurrent: 8})
if limiter.Acquire() != "" || limiter.Acquire() != "" {
t.Fatal("two slots should be free")
}
if got := limiter.Acquire(); got != RefuseConcurrency {
t.Fatalf("third acquire: %q", got)
}
limiter.Release()
if got := limiter.Acquire(); got != "" {
t.Fatalf("after release: %q", got)
}
}
func TestServerCanNarrowConcurrency(t *testing.T) {
cfg := DefaultConfig()
cfg.MaxConcurrent = 4
limiter, _ := testLimiter(t, cfg, "2026-10-15T12:00:00Z")
limiter.ApplyServer(&ServerLimits{MaxConcurrentPerNode: 2}, &ServerSettings{MaxConcurrent: 1})
if limiter.Acquire() != "" {
t.Fatal("first slot")
}
if got := limiter.Acquire(); got != RefuseConcurrency {
t.Fatalf("got %q", got)
}
}
func TestBandwidthCapLocalAndDayRollover(t *testing.T) {
cfg := DefaultConfig()
cfg.BandwidthMBDay = 1
limiter, c := testLimiter(t, cfg, "2026-10-15T23:00:00Z")
limiter.ApplyServer(&ServerLimits{MaxDonorMBPerDay: 20480}, &ServerSettings{BandwidthMBDay: 9999})
if limiter.Acquire() != "" {
t.Fatal("fresh day must allow")
}
limiter.Release()
limiter.AddBytes(mebibyte - 10)
if got := limiter.Remaining(); got != 10 {
t.Fatalf("remaining %d", got)
}
limiter.AddBytes(10)
if got := limiter.Acquire(); got != RefuseBandwidth {
t.Fatalf("got %q", got)
}
c.now = c.now.Add(2 * time.Hour)
if got := limiter.Acquire(); got != "" {
t.Fatalf("next day: %q", got)
}
}
func TestServerCanNarrowBandwidth(t *testing.T) {
cfg := DefaultConfig()
cfg.BandwidthMBDay = 500
limiter, _ := testLimiter(t, cfg, "2026-10-15T12:00:00Z")
limiter.ApplyServer(nil, &ServerSettings{BandwidthMBDay: 2})
if got := limiter.Remaining(); got != 2*mebibyte {
t.Fatalf("remaining %d", got)
}
}
func TestActiveHoursLocalAndServer(t *testing.T) {
cfg := DefaultConfig()
cfg.ActiveFrom, cfg.ActiveTo = "22:00", "06:00"
limiter, c := testLimiter(t, cfg, "2026-10-15T12:00:00Z")
if got := limiter.Acquire(); got != RefuseHours {
t.Fatalf("noon: %q", got)
}
c.now = c.now.Add(11 * time.Hour)
if got := limiter.Acquire(); got != "" {
t.Fatalf("23:00: %q", got)
}
limiter.Release()
limiter.ApplyServer(nil, &ServerSettings{ActiveFrom: "01:00", ActiveTo: "02:00", Timezone: "UTC"})
if got := limiter.Acquire(); got != RefuseHours {
t.Fatalf("server window: %q", got)
}
}
func TestUsagePersistsAcrossRestart(t *testing.T) {
path := filepath.Join(t.TempDir(), "usage.json")
cfg := DefaultConfig()
cfg.Timezone = "UTC"
moment := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)
c := &clock{now: moment}
first, err := NewLimiter(cfg, path, c.Now)
if err != nil {
t.Fatal(err)
}
first.AddBytes(12345)
first.Flush()
second, err := NewLimiter(cfg, path, c.Now)
if err != nil {
t.Fatal(err)
}
if _, used, _ := second.Usage(); used != 12345 {
t.Fatalf("used %d", used)
}
c.now = moment.Add(24 * time.Hour)
if _, used, _ := second.Usage(); used != 0 {
t.Fatalf("next day used %d", used)
}
if got := ReadUsage(path, time.UTC, moment); got != 12345 {
t.Fatalf("read usage %d", got)
}
}
nodes/client/logger.go
+26
View File
@@ -0,0 +1,26 @@
package main
import (
"fmt"
"io"
"log"
)
type Logger struct {
out *log.Logger
verbose bool
}
func NewLogger(w io.Writer, verbose bool) *Logger {
return &Logger{out: log.New(w, "devplace-node ", log.LstdFlags|log.LUTC), verbose: verbose}
}
func (l *Logger) Printf(format string, args ...any) {
l.out.Output(2, fmt.Sprintf(format, args...))
}
func (l *Logger) Debugf(format string, args ...any) {
if l.verbose {
l.out.Output(2, fmt.Sprintf(format, args...))
}
}
nodes/client/main.go
+211
View File
@@ -0,0 +1,211 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"io"
"os"
"os/signal"
"path/filepath"
"syscall"
"time"
_ "time/tzdata"
)
var Version = "dev"
const (
exitOK = 0
exitError = 1
exitUsage = 2
exitFatal = 3
usageText = `usage: devplace-node <command> [flags]
commands:
login store the server and device secret (--server URL --secret dpn_...)
status show the stored configuration, today's usage and whether the server accepts this device
run connect and serve jobs in the foreground
version print the version
flags for every command: --config PATH (default: <user config dir>/devplace-node/config.json)
`
)
func main() {
os.Exit(execute(os.Args[1:], os.Stdout, os.Stderr))
}
func execute(args []string, stdout, stderr io.Writer) int {
if len(args) == 0 {
fmt.Fprint(stderr, usageText)
return exitUsage
}
command, rest := args[0], args[1:]
switch command {
case "version":
fmt.Fprintln(stdout, "devplace-node", Version)
return exitOK
case "login":
return runLogin(rest, stdout, stderr)
case "status":
return runStatus(rest, stdout, stderr)
case "run":
return runRun(rest, stderr)
case "help", "-h", "--help":
fmt.Fprint(stdout, usageText)
return exitOK
}
fmt.Fprintf(stderr, "unknown command %q\n%s", command, usageText)
return exitUsage
}
func resolveConfigPath(flagValue string) (string, error) {
if flagValue != "" {
return flagValue, nil
}
return DefaultConfigPath()
}
func newFlags(name string, stderr io.Writer) (*flag.FlagSet, *string) {
set := flag.NewFlagSet(name, flag.ContinueOnError)
set.SetOutput(stderr)
path := set.String("config", "", "config file path")
return set, path
}
func runLogin(args []string, stdout, stderr io.Writer) int {
set, pathFlag := newFlags("login", stderr)
server := set.String("server", "", "server url, for example https://devplace.net")
secret := set.String("secret", "", "device secret (or set DEVPLACE_NODE_SECRET)")
bandwidth := set.Int("bandwidth-mb", DefaultBandwidthMB, "daily bandwidth cap in MB")
concurrent := set.Int("concurrency", DefaultMaxConcurrent, "maximum parallel jobs")
tunnels := set.Int("tunnels", DefaultMaxTunnels, "maximum parallel tunnels")
from := set.String("from", "", "active hours start HH:MM")
to := set.String("to", "", "active hours end HH:MM")
zone := set.String("timezone", "", "IANA timezone for the active hours (default: system)")
skip := set.Bool("no-verify", false, "store without asking the server to accept the secret")
if err := set.Parse(args); err != nil {
return exitUsage
}
if *secret == "" {
*secret = os.Getenv("DEVPLACE_NODE_SECRET")
}
path, err := resolveConfigPath(*pathFlag)
if err != nil {
fmt.Fprintln(stderr, err)
return exitError
}
config := Config{
Server: *server, Secret: *secret, BandwidthMBDay: *bandwidth, MaxConcurrent: *concurrent,
MaxTunnels: *tunnels, ActiveFrom: *from, ActiveTo: *to, Timezone: *zone,
}
if err := config.Validate(); err != nil {
fmt.Fprintln(stderr, "login refused:", err)
return exitUsage
}
if !*skip {
if code := verifyDevice(config, stderr); code != exitOK {
return code
}
}
if err := SaveConfig(path, config); err != nil {
fmt.Fprintln(stderr, "config could not be written:", err)
return exitError
}
fmt.Fprintf(stdout, "stored %s (mode 0600)\n", path)
return exitOK
}
func verifyDevice(config Config, stderr io.Writer) int {
session := NewSession(config, NewPolicy(), nil, nil, nil, NewLogger(stderr, false))
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(TicketTimeoutSeconds+5)*time.Second)
defer cancel()
_, result := session.fetchTicket(ctx)
if result.fatal != "" {
fmt.Fprintln(stderr, "server refused this device:", result.fatal)
return exitFatal
}
if result.reason != "" {
fmt.Fprintln(stderr, "server not verified:", result.reason, "(use --no-verify to store anyway)")
return exitError
}
return exitOK
}
func runStatus(args []string, stdout, stderr io.Writer) int {
set, pathFlag := newFlags("status", stderr)
if err := set.Parse(args); err != nil {
return exitUsage
}
path, err := resolveConfigPath(*pathFlag)
if err != nil {
fmt.Fprintln(stderr, err)
return exitError
}
config, err := LoadConfig(path)
if err != nil {
fmt.Fprintln(stderr, err)
return exitError
}
location, _ := config.Location()
used := ReadUsage(filepath.Join(filepath.Dir(path), UsageFileName), location, time.Now())
fmt.Fprintf(stdout, "version: %s\nconfig: %s\nserver: %s\nsecret: stored\n", Version, path, config.Server)
fmt.Fprintf(stdout, "bandwidth: %d MB/day (used today: %.1f MB)\n", config.BandwidthMBDay, float64(used)/mebibyte)
fmt.Fprintf(stdout, "concurrency: %d jobs, %d tunnels\n", config.MaxConcurrent, config.MaxTunnels)
hours := "always"
if config.ActiveFrom != "" {
hours = config.ActiveFrom + "-" + config.ActiveTo
}
fmt.Fprintf(stdout, "active hours: %s (%s)\n", hours, location)
if code := verifyDevice(config, stderr); code != exitOK {
fmt.Fprintln(stdout, "server: not accepting this device")
return code
}
fmt.Fprintln(stdout, "server: accepts this device")
return exitOK
}
func runRun(args []string, stderr io.Writer) int {
set, pathFlag := newFlags("run", stderr)
verbose := set.Bool("v", false, "verbose logging")
if err := set.Parse(args); err != nil {
return exitUsage
}
path, err := resolveConfigPath(*pathFlag)
if err != nil {
fmt.Fprintln(stderr, err)
return exitError
}
config, err := LoadConfig(path)
if err != nil {
fmt.Fprintln(stderr, err)
return exitError
}
logger := NewLogger(stderr, *verbose)
limiter, err := NewLimiter(config, filepath.Join(filepath.Dir(path), UsageFileName), time.Now)
if err != nil {
fmt.Fprintln(stderr, err)
return exitError
}
policy := NewPolicy()
stats := NewStats(time.Now())
session := NewSession(config, policy, limiter, NewFetcher(policy), stats, logger)
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
logger.Printf("started version %s, server %s", Version, config.Server)
err = session.Run(ctx)
limiter.Flush()
logger.Printf("%s", stats.Line())
var fatal *FatalError
if errors.As(err, &fatal) {
logger.Printf("%s", fatal.Error())
return exitFatal
}
if err != nil {
logger.Printf("stopped: %v", err)
return exitError
}
return exitOK
}
nodes/client/packaging/compose.yaml
+15
View File
@@ -0,0 +1,15 @@
services:
devplace-node:
image: gcr.io/distroless/static-debian12:nonroot
command: ["/devplace-node", "run", "--config", "/config/config.json"]
volumes:
- ./devplace-node-linux-amd64:/devplace-node:ro
- ./config:/config
restart: on-failure
read_only: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
mem_limit: 128m
pids_limit: 64
nodes/client/packaging/devplace-node.service
+16
View File
@@ -0,0 +1,16 @@
[Unit]
Description=DevPlace community node
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
ExecStart=%h/.local/bin/devplace-node run
Restart=on-failure
RestartSec=30
RestartPreventExitStatus=3
Nice=10
NoNewPrivileges=true
[Install]
WantedBy=default.target
nodes/client/packaging/nl.devplace.node.plist
+32
View File
@@ -0,0 +1,32 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>nl.devplace.node</string>
<key>ProgramArguments</key>
<array>
<string>/usr/local/bin/devplace-node</string>
<string>run</string>
</array>
<key>RunAtLoad</key>
<true/>
<key>KeepAlive</key>
<dict>
<key>SuccessfulExit</key>
<false/>
</dict>
<key>ThrottleInterval</key>
<integer>60</integer>
<key>ProcessType</key>
<string>Background</string>
<key>LowPriorityIO</key>
<true/>
<key>Nice</key>
<integer>10</integer>
<key>StandardErrorPath</key>
<string>/tmp/devplace-node.log</string>
<key>StandardOutPath</key>
<string>/dev/null</string>
</dict>
</plist>
nodes/client/packaging/windows-install.ps1
+34
View File
@@ -0,0 +1,34 @@
param(
[string]$Binary = "$env:LOCALAPPDATA\devplace-node\devplace-node.exe",
[switch]$Remove
)
$ErrorActionPreference = "Stop"
$TaskName = "DevPlaceNode"
if ($Remove) {
Unregister-ScheduledTask -TaskName $TaskName -Confirm:$false
Write-Host "Removed scheduled task $TaskName"
exit 0
}
if (-not (Test-Path $Binary)) {
throw "devplace-node.exe not found at $Binary. Download it there first."
}
$action = New-ScheduledTaskAction -Execute $Binary -Argument "run"
$trigger = New-ScheduledTaskTrigger -AtLogOn -User $env:USERNAME
$settings = New-ScheduledTaskSettingsSet `
-AllowStartIfOnBatteries `
-DontStopIfGoingOnBatteries `
-RestartCount 3 `
-RestartInterval (New-TimeSpan -Minutes 5) `
-ExecutionTimeLimit ([TimeSpan]::Zero) `
-StartWhenAvailable
$principal = New-ScheduledTaskPrincipal -UserId $env:USERNAME -LogonType Interactive -RunLevel Limited
Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger $trigger -Settings $settings -Principal $principal -Force | Out-Null
Write-Host "Registered scheduled task $TaskName. It starts at your next logon."
Write-Host "Store the device secret first: devplace-node.exe login --server https://devplace.net --secret dpn_..."
Write-Host "Start now: Start-ScheduledTask -TaskName $TaskName"
Write-Host "Stop and remove: .\windows-install.ps1 -Remove"
nodes/client/policy.go
+317
View File
@@ -0,0 +1,317 @@
package main
import (
"context"
"errors"
"fmt"
"net"
"net/netip"
"net/url"
"slices"
"strconv"
"strings"
"sync"
"time"
)
type PolicyError struct {
Code string
Reason string
}
func (e *PolicyError) Error() string {
if e.Reason == "" {
return e.Code
}
return e.Code + ": " + e.Reason
}
func refuse(code, reason string) error {
return &PolicyError{Code: code, Reason: reason}
}
type Target struct {
URL string
Method string
Host string
Port int
Dest Destination
}
type Policy struct {
mu sync.RWMutex
dests []Destination
lookup func(ctx context.Context, host string) ([]netip.Addr, error)
public func(netip.Addr) bool
dial func(ctx context.Context, network, address string) (net.Conn, error)
}
func NewPolicy() *Policy {
dialer := &net.Dialer{Timeout: time.Duration(TunnelDialSeconds) * time.Second}
return &Policy{
dests: cloneDestinations(BakedDestinations),
lookup: systemLookup,
public: IsPublicAddr,
dial: dialer.DialContext,
}
}
func systemLookup(ctx context.Context, host string) ([]netip.Addr, error) {
return net.DefaultResolver.LookupNetIP(ctx, "ip", host)
}
func cloneDestinations(list []Destination) []Destination {
out := make([]Destination, len(list))
for i, item := range list {
item.Methods = slices.Clone(item.Methods)
item.Ports = slices.Clone(item.Ports)
item.Modes = slices.Clone(item.Modes)
out[i] = item
}
return out
}
func normalizeHost(host string) string {
return strings.TrimRight(strings.ToLower(strings.TrimSpace(host)), ".")
}
func suffixMatches(suffix, host string) bool {
return host == suffix || strings.HasSuffix(host, "."+suffix)
}
func intersectStrings(a, b []string) []string {
out := []string{}
for _, item := range a {
if slices.ContainsFunc(b, func(other string) bool { return strings.EqualFold(item, other) }) {
out = append(out, item)
}
}
return out
}
func intersectPorts(a, b []int) []int {
out := []int{}
for _, item := range a {
if slices.Contains(b, item) && slices.Contains(AllowedPorts, item) {
out = append(out, item)
}
}
return out
}
func (p *Policy) Refine(offered []Destination) []string {
var rejected []string
narrowed := []Destination{}
seen := map[string]bool{}
for _, item := range offered {
suffix := normalizeHost(item.Suffix)
var base *Destination
for i := range BakedDestinations {
if suffixMatches(BakedDestinations[i].Suffix, suffix) {
base = &BakedDestinations[i]
break
}
}
if base == nil || seen[suffix] {
rejected = append(rejected, item.Suffix)
continue
}
seen[suffix] = true
narrowed = append(narrowed, Destination{
Suffix: suffix,
Adapter: base.Adapter,
Methods: intersectStrings(base.Methods, item.Methods),
Ports: intersectPorts(base.Ports, item.Ports),
Modes: intersectStrings(base.Modes, item.Modes),
MaxRPS: item.MaxRPS,
})
}
p.mu.Lock()
p.dests = narrowed
p.mu.Unlock()
return rejected
}
func (p *Policy) Destinations() []Destination {
p.mu.RLock()
defer p.mu.RUnlock()
return cloneDestinations(p.dests)
}
func (p *Policy) destinationFor(host string) (Destination, bool) {
p.mu.RLock()
defer p.mu.RUnlock()
var best Destination
found := false
for _, item := range p.dests {
if suffixMatches(item.Suffix, host) && (!found || len(item.Suffix) > len(best.Suffix)) {
best, found = item, true
}
}
return best, found
}
func isAddressLiteral(host string) bool {
_, err := netip.ParseAddr(strings.Trim(host, "[]"))
return err == nil
}
func (p *Policy) CheckHostPort(host string, port int, mode string) (Destination, error) {
host = normalizeHost(host)
if host == "" {
return Destination{}, refuse(RefuseBadURL, "no host")
}
if isAddressLiteral(host) {
return Destination{}, refuse(RefuseHost, "address literals are not allowed")
}
dest, ok := p.destinationFor(host)
if !ok {
return Destination{}, refuse(RefuseHost, "destination is not on the allowlist")
}
if !slices.Contains(AllowedPorts, port) || !slices.Contains(dest.Ports, port) {
return Destination{}, refuse(RefusePort, "port is not allowed")
}
if mode != "" && mode != "any" && !slices.Contains(dest.Modes, mode) {
return Destination{}, refuse(RefuseHost, "mode is not allowed for this destination")
}
return dest, nil
}
func (p *Policy) Check(rawURL, method, mode string) (Target, error) {
if rawURL == "" || len(rawURL) > MaxURLChars {
return Target{}, refuse(RefuseBadURL, "url is missing or too long")
}
parsed, err := url.Parse(strings.TrimSpace(rawURL))
if err != nil {
return Target{}, refuse(RefuseBadURL, "url cannot be parsed")
}
defaultPort := 0
switch parsed.Scheme {
case "http":
defaultPort = 80
case "https":
defaultPort = 443
default:
return Target{}, refuse(RefuseBadURL, "only http and https are allowed")
}
if parsed.User != nil {
return Target{}, refuse(RefuseBadURL, "credentials in a url are not allowed")
}
port := defaultPort
if text := parsed.Port(); text != "" {
port, err = strconv.Atoi(text)
if err != nil {
return Target{}, refuse(RefuseBadURL, "port is not a number")
}
}
dest, err := p.CheckHostPort(parsed.Hostname(), port, mode)
if err != nil {
return Target{}, err
}
verb := strings.ToUpper(strings.TrimSpace(method))
if !slices.Contains(upperAll(dest.Methods), verb) {
return Target{}, refuse(RefuseMethod, "method is not allowed here")
}
return Target{URL: parsed.String(), Method: verb, Host: normalizeHost(parsed.Hostname()), Port: port, Dest: dest}, nil
}
func upperAll(list []string) []string {
out := make([]string, len(list))
for i, item := range list {
out[i] = strings.ToUpper(item)
}
return out
}
func (p *Policy) CheckRedirect(location, method string) (Target, error) {
target, err := p.Check(location, method, "fetch")
if err != nil {
var policyErr *PolicyError
if errors.As(err, &policyErr) {
return Target{}, refuse(RefuseRedirect, policyErr.Reason)
}
return Target{}, refuse(RefuseRedirect, err.Error())
}
return target, nil
}
func (p *Policy) Resolve(ctx context.Context, host string) ([]netip.Addr, error) {
host = normalizeHost(host)
addrs, err := p.lookup(ctx, host)
if err != nil {
return nil, refuse(FailDNS, "host did not resolve")
}
if len(addrs) == 0 {
return nil, refuse(FailDNS, "host has no addresses")
}
if len(addrs) > 16 {
addrs = addrs[:16]
}
for _, addr := range addrs {
if !p.public(addr.Unmap()) {
return nil, refuse(RefuseAddress, "host resolves to a non public address")
}
}
return addrs, nil
}
func (p *Policy) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
host, portText, err := net.SplitHostPort(address)
if err != nil {
return nil, refuse(RefuseBadURL, "bad address")
}
port, err := strconv.Atoi(portText)
if err != nil {
return nil, refuse(RefuseBadURL, "bad port")
}
if _, err := p.CheckHostPort(host, port, ""); err != nil {
return nil, err
}
addrs, err := p.Resolve(ctx, host)
if err != nil {
return nil, err
}
var last error
for _, addr := range addrs {
conn, err := p.dial(ctx, network, net.JoinHostPort(addr.Unmap().String(), portText))
if err == nil {
return conn, nil
}
last = err
}
return nil, fmt.Errorf("connect failed: %w", last)
}
var blockedPrefixes = mustPrefixes(
"0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8", "169.254.0.0/16",
"172.16.0.0/12", "192.0.0.0/24", "192.0.2.0/24", "192.88.99.0/24", "192.168.0.0/16",
"198.18.0.0/15", "198.51.100.0/24", "203.0.113.0/24", "224.0.0.0/4", "240.0.0.0/4",
"::/128", "::1/128", "::ffff:0:0/96", "64:ff9b::/96", "64:ff9b:1::/48", "100::/64",
"2001::/32", "2001:db8::/32", "2002::/16", "fc00::/7", "fe80::/10", "ff00::/8",
)
func mustPrefixes(list ...string) []netip.Prefix {
out := make([]netip.Prefix, len(list))
for i, item := range list {
out[i] = netip.MustParsePrefix(item)
}
return out
}
func IsPublicAddr(addr netip.Addr) bool {
addr = addr.Unmap()
if !addr.IsValid() || addr.Zone() != "" {
return false
}
if addr.IsUnspecified() || addr.IsLoopback() || addr.IsPrivate() || addr.IsMulticast() ||
addr.IsLinkLocalUnicast() || addr.IsLinkLocalMulticast() || addr.IsInterfaceLocalMulticast() ||
!addr.IsGlobalUnicast() {
return false
}
for _, prefix := range blockedPrefixes {
if prefix.Contains(addr) {
return false
}
}
return true
}
nodes/client/policy_test.go
+233
View File
@@ -0,0 +1,233 @@
package main
import (
"context"
"errors"
"net"
"net/netip"
"testing"
)
func testPolicy(answer func(host string) []netip.Addr) *Policy {
policy := NewPolicy()
policy.lookup = func(ctx context.Context, host string) ([]netip.Addr, error) {
return answer(host), nil
}
return policy
}
func publicAnswer(string) []netip.Addr {
return []netip.Addr{netip.MustParseAddr("93.184.216.34")}
}
func codeOf(err error) string {
var policyErr *PolicyError
if errors.As(err, &policyErr) {
return policyErr.Code
}
if err == nil {
return ""
}
return "other:" + err.Error()
}
func TestCheckTable(t *testing.T) {
policy := testPolicy(publicAnswer)
cases := []struct {
name string
url string
method string
mode string
want string
}{
{"apex", "https://reddit.com/r/go", "GET", "fetch", ""},
{"subdomain", "https://old.reddit.com/r/go.json", "get", "fetch", ""},
{"short domain", "https://redd.it/abc", "GET", "any", ""},
{"explicit 443", "https://www.reddit.com:443/", "GET", "fetch", ""},
{"plain http 80", "http://reddit.com/", "GET", "fetch", ""},
{"trailing dot", "https://reddit.com./", "GET", "fetch", ""},
{"upper case host", "https://WWW.REDDIT.COM/", "GET", "fetch", ""},
{"prefix attack", "https://evilreddit.com/", "GET", "fetch", RefuseHost},
{"suffix attack", "https://reddit.com.evil.net/", "GET", "fetch", RefuseHost},
{"hyphen attack", "https://not-reddit.com/", "GET", "fetch", RefuseHost},
{"other host", "https://example.com/", "GET", "fetch", RefuseHost},
{"port 8080", "https://reddit.com:8080/", "GET", "fetch", RefusePort},
{"port 22", "http://reddit.com:22/", "GET", "fetch", RefusePort},
{"port 0", "https://reddit.com:0/", "GET", "fetch", RefusePort},
{"ipv4 literal", "http://127.0.0.1/", "GET", "fetch", RefuseHost},
{"ipv4 literal 80", "http://93.184.216.34/", "GET", "fetch", RefuseHost},
{"ipv6 literal", "http://[::1]/", "GET", "fetch", RefuseHost},
{"credentials", "https://user:pw@reddit.com/", "GET", "fetch", RefuseBadURL},
{"scheme ftp", "ftp://reddit.com/", "GET", "fetch", RefuseBadURL},
{"scheme file", "file:///etc/passwd", "GET", "fetch", RefuseBadURL},
{"empty", "", "GET", "fetch", RefuseBadURL},
{"post", "https://reddit.com/", "POST", "fetch", RefuseMethod},
{"delete", "https://reddit.com/", "DELETE", "fetch", RefuseMethod},
{"bad port text", "https://reddit.com:abc/", "GET", "fetch", RefuseBadURL},
}
for _, item := range cases {
_, err := policy.Check(item.url, item.method, item.mode)
if got := codeOf(err); got != item.want {
t.Errorf("%s: got %q want %q", item.name, got, item.want)
}
}
long := "https://reddit.com/" + string(make([]byte, MaxURLChars))
if _, err := policy.Check(long, "GET", "fetch"); codeOf(err) != RefuseBadURL {
t.Errorf("long url not refused: %v", err)
}
}
func TestCheckHostPortModes(t *testing.T) {
policy := testPolicy(publicAnswer)
if _, err := policy.CheckHostPort("reddit.com", 443, "tunnel"); err != nil {
t.Fatal(err)
}
if _, err := policy.CheckHostPort("reddit.com", 25, "tunnel"); codeOf(err) != RefusePort {
t.Fatalf("port: %v", err)
}
if _, err := policy.CheckHostPort("evilreddit.com", 443, "tunnel"); codeOf(err) != RefuseHost {
t.Fatalf("host: %v", err)
}
}
func TestPublicAddressTable(t *testing.T) {
cases := map[string]bool{
"8.8.8.8": true, "93.184.216.34": true, "2606:4700:4700::1111": true,
"10.0.0.1": false, "10.255.255.255": false, "127.0.0.1": false, "127.1.2.3": false,
"169.254.169.254": false, "172.16.0.1": false, "172.31.255.255": false, "172.32.0.1": true,
"192.168.1.1": false, "100.64.0.1": false, "100.127.255.255": false, "100.128.0.1": true,
"0.0.0.0": false, "0.1.2.3": false, "224.0.0.1": false, "239.255.255.250": false,
"255.255.255.255": false, "240.0.0.1": false, "198.18.0.1": false, "192.0.2.1": false,
"::": false, "::1": false, "fe80::1": false, "fc00::1": false, "fd12:3456::1": false,
"ff02::1": false, "::ffff:127.0.0.1": false, "::ffff:10.0.0.1": false, "::ffff:8.8.8.8": true,
"2001:db8::1": false, "64:ff9b::7f00:1": false, "2002:7f00:1::1": false, "2001::1": false,
}
for text, want := range cases {
if got := IsPublicAddr(netip.MustParseAddr(text)); got != want {
t.Errorf("%s: got %v want %v", text, got, want)
}
}
if IsPublicAddr(netip.Addr{}) {
t.Error("invalid address counted as public")
}
if IsPublicAddr(netip.MustParseAddr("fe80::1%eth0")) {
t.Error("zoned address counted as public")
}
}
func TestResolveRefusesAnyPrivateAnswer(t *testing.T) {
cases := map[string][]string{
"loopback": {"127.0.0.1"},
"private": {"10.1.2.3"},
"mixed": {"93.184.216.34", "192.168.0.1"},
"v6 loopback": {"::1"},
"v6 mapped": {"::ffff:10.0.0.1"},
"cgnat": {"100.64.5.5"},
}
for name, list := range cases {
addrs := []netip.Addr{}
for _, item := range list {
addrs = append(addrs, netip.MustParseAddr(item))
}
policy := testPolicy(func(string) []netip.Addr { return addrs })
if _, err := policy.Resolve(context.Background(), "reddit.com"); codeOf(err) != RefuseAddress {
t.Errorf("%s: %v", name, err)
}
}
empty := testPolicy(func(string) []netip.Addr { return nil })
if _, err := empty.Resolve(context.Background(), "reddit.com"); codeOf(err) != FailDNS {
t.Errorf("empty answer: %v", err)
}
}
func TestDialConnectsToValidatedIPAndDefeatsRebinding(t *testing.T) {
calls := 0
policy := testPolicy(func(string) []netip.Addr {
calls++
if calls == 1 {
return []netip.Addr{netip.MustParseAddr("93.184.216.34")}
}
return []netip.Addr{netip.MustParseAddr("127.0.0.1")}
})
var dialed []string
policy.dial = func(ctx context.Context, network, address string) (net.Conn, error) {
dialed = append(dialed, address)
client, server := net.Pipe()
server.Close()
return client, nil
}
if _, err := policy.DialContext(context.Background(), "tcp", "www.reddit.com:443"); err != nil {
t.Fatal(err)
}
if len(dialed) != 1 || dialed[0] != "93.184.216.34:443" {
t.Fatalf("dialed %v", dialed)
}
if _, err := policy.DialContext(context.Background(), "tcp", "www.reddit.com:443"); codeOf(err) != RefuseAddress {
t.Fatalf("rebinding not refused: %v", err)
}
if len(dialed) != 1 {
t.Fatalf("private address was dialed: %v", dialed)
}
if _, err := policy.DialContext(context.Background(), "tcp", "evil.com:443"); codeOf(err) != RefuseHost {
t.Fatalf("host: %v", err)
}
if _, err := policy.DialContext(context.Background(), "tcp", "reddit.com:22"); codeOf(err) != RefusePort {
t.Fatalf("port: %v", err)
}
}
func TestRedirectLeavingAllowlistRefused(t *testing.T) {
policy := testPolicy(publicAnswer)
cases := map[string]string{
"https://old.reddit.com/x": "",
"https://redd.it/y": "",
"https://evil.com/": RefuseRedirect,
"https://evilreddit.com/": RefuseRedirect,
"http://reddit.com:8080/": RefuseRedirect,
"http://127.0.0.1/": RefuseRedirect,
"ftp://reddit.com/": RefuseRedirect,
"https://reddit.com.evil.io": RefuseRedirect,
}
for location, want := range cases {
_, err := policy.CheckRedirect(location, "GET")
if got := codeOf(err); got != want {
t.Errorf("%s: got %q want %q", location, got, want)
}
}
if _, err := policy.CheckRedirect("https://reddit.com/", "POST"); codeOf(err) != RefuseRedirect {
t.Error("method change through redirect allowed")
}
}
func TestRefineOnlyNarrows(t *testing.T) {
policy := testPolicy(publicAnswer)
rejected := policy.Refine([]Destination{
{Suffix: "old.reddit.com", Methods: []string{"GET", "POST"}, Ports: []int{80, 443, 8080}, Modes: []string{"fetch", "any"}},
{Suffix: "redd.it", Methods: []string{"GET"}, Ports: []int{443}, Modes: []string{"tunnel", "fetch"}},
{Suffix: "evil.com", Methods: []string{"GET"}, Ports: []int{443}, Modes: []string{"fetch"}},
{Suffix: "evilreddit.com", Methods: []string{"GET"}, Ports: []int{443}, Modes: []string{"fetch"}},
{Suffix: "old.reddit.com", Methods: []string{"GET"}, Ports: []int{443}, Modes: []string{"fetch"}},
})
if len(rejected) != 3 {
t.Fatalf("rejected %v", rejected)
}
if _, err := policy.Check("https://old.reddit.com/", "GET", "fetch"); err != nil {
t.Fatal(err)
}
if _, err := policy.Check("https://old.reddit.com/", "POST", "fetch"); codeOf(err) != RefuseMethod {
t.Errorf("post widened: %v", err)
}
if _, err := policy.Check("http://old.reddit.com:8080/", "GET", "fetch"); codeOf(err) != RefusePort {
t.Errorf("port widened: %v", err)
}
if _, err := policy.Check("https://www.reddit.com/", "GET", "fetch"); codeOf(err) != RefuseHost {
t.Errorf("dropped destination still allowed: %v", err)
}
if _, err := policy.Check("http://redd.it/a", "GET", "fetch"); codeOf(err) != RefusePort {
t.Errorf("redd.it port 80 should be gone: %v", err)
}
policy.Refine(nil)
if _, err := policy.Check("https://redd.it/a", "GET", "fetch"); codeOf(err) != RefuseHost {
t.Errorf("empty refinement must allow nothing: %v", err)
}
}
nodes/client/protocol.go
+146
View File
@@ -0,0 +1,146 @@
package main
import "encoding/json"
const (
ProtocolVersion = 1
DeviceKind = "native"
SecretPrefix = "dpn_"
MaxWSFramePayload = 256 * 1024
MaxInboundMessage = 256 * 1024
MaxResultMessage = 3 * 1024 * 1024
DefaultReconnectMin = 1
DefaultReconnectMax = 60
DefaultPingSeconds = 20
DefaultMaxBodyBytes = 2 * 1024 * 1024
DefaultJobTimeout = 25
MaxJobTimeout = 60
HelloTimeoutSeconds = 10
TicketTimeoutSeconds = 15
MissedPingsBeforeDead = 3
UsageIntervalSeconds = 60
MaxRedirects = 5
MaxRequestHeaders = 32
MaxResponseHeaders = 64
MaxHeaderValueBytes = 1024
MaxIDBytes = 64
MaxURLChars = 2048
MaxConcurrentCeiling = 8
MaxTunnelsCeiling = 8
TunnelChunkBytes = 32 * 1024
TunnelQueueDepth = 64
TunnelIdleSeconds = 60
TunnelLifetimeSeconds = 300
TunnelDialSeconds = 10
MaxDonorMBPerDay = 20 * 1024
StableSessionSeconds = 30
FastRetryMillis = 1500
)
const (
RefuseHost = "host_not_allowed"
RefusePort = "port_not_allowed"
RefuseAddress = "address_not_public"
RefuseRedirect = "redirect_not_allowed"
RefuseMethod = "method_not_allowed"
RefuseBandwidth = "bandwidth_exhausted"
RefuseConcurrency = "concurrency_exceeded"
RefuseHours = "outside_hours"
RefuseTimeout = "timeout"
RefuseBadURL = "bad_url"
FailFetch = "fetch_failed"
FailDNS = "dns_failed"
)
const (
CloseNormal = 1000
CloseGoingAway = 1001
CloseProtocolError = 1002
CloseUnsupported = 1003
CloseInvalidPayload = 1007
ClosePolicy = 1008
CloseTooLarge = 1009
CloseTryAgain = 1013
CloseRetryOwner = 4013
CloseCapacity = 4029
CloseProtocol = 4400
CloseKicked = 4403
)
var FatalReasons = map[string]bool{
"revoked": true,
"suspended": true,
"kill_switch": true,
"no_consent": true,
"no_account": true,
"deletion_pending": true,
"account_inactive": true,
"not_adult": true,
"not_enabled": true,
"replaced": true,
"device": true,
}
var BakedDestinations = []Destination{
{Suffix: "reddit.com", Adapter: "reddit", Methods: []string{"GET"}, Ports: []int{80, 443}, Modes: []string{"tunnel", "fetch"}},
{Suffix: "redd.it", Adapter: "reddit", Methods: []string{"GET"}, Ports: []int{80, 443}, Modes: []string{"tunnel", "fetch"}},
}
var AllowedPorts = []int{80, 443}
type Destination struct {
Suffix string `json:"suffix"`
Adapter string `json:"adapter"`
Methods []string `json:"methods"`
Ports []int `json:"ports"`
Modes []string `json:"modes"`
MaxRPS float64 `json:"max_rps"`
}
type ServerLimits struct {
MaxBodyBytes int `json:"max_body_bytes"`
MaxConcurrentPerNode int `json:"max_concurrent_per_node"`
MaxDonorMBPerDay int `json:"max_donor_mb_per_day"`
PingSeconds int `json:"ping_seconds"`
ReconnectMinSeconds int `json:"reconnect_min_seconds"`
ReconnectMaxSeconds int `json:"reconnect_max_seconds"`
JobTimeoutSeconds int `json:"job_timeout_seconds"`
MaxFrameBytes int `json:"max_frame_bytes"`
}
type ServerSettings struct {
Enabled bool `json:"enabled"`
BandwidthMBDay int `json:"bandwidth_mb_day"`
MaxConcurrent int `json:"max_concurrent"`
ActiveFrom string `json:"active_from"`
ActiveTo string `json:"active_to"`
Timezone string `json:"timezone"`
}
type Frame struct {
Type string `json:"type"`
JobID string `json:"job_id"`
Method string `json:"method"`
URL string `json:"url"`
Headers map[string]any `json:"headers"`
Timeout int `json:"timeout"`
ConnID string `json:"conn_id"`
Host string `json:"host"`
Port int `json:"port"`
B64 string `json:"b64"`
Reason string `json:"reason"`
DeviceUID string `json:"device_uid"`
Limits *ServerLimits `json:"limits"`
Destinations []Destination `json:"destinations"`
Settings *ServerSettings `json:"settings"`
}
func decodeFrame(raw []byte) (Frame, error) {
var frame Frame
if err := json.Unmarshal(raw, &frame); err != nil {
return frame, err
}
return frame, nil
}
nodes/client/session.go
+445
View File
@@ -0,0 +1,445 @@
package main
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"math/rand"
"net/http"
"net/url"
"runtime"
"strings"
"sync"
"time"
)
type FatalError struct {
Reason string
}
func (e *FatalError) Error() string {
return "the server ended this device for good: " + e.Reason
}
type outcome struct {
fatal string
fast bool
stable bool
reason string
}
type Session struct {
cfg Config
policy *Policy
limiter *Limiter
fetcher *Fetcher
stats *Stats
log *Logger
httpc *http.Client
jitter func() float64
dialWS func(ctx context.Context, rawURL string) (*Conn, error)
limits ServerLimits
maxBody int
}
func NewSession(cfg Config, policy *Policy, limiter *Limiter, fetcher *Fetcher, stats *Stats, logger *Logger) *Session {
return &Session{
cfg: cfg,
policy: policy,
limiter: limiter,
fetcher: fetcher,
stats: stats,
log: logger,
httpc: &http.Client{Timeout: time.Duration(TicketTimeoutSeconds) * time.Second, Transport: &http.Transport{Proxy: http.ProxyFromEnvironment}},
jitter: rand.Float64,
dialWS: Dial,
limits: defaultLimits(),
maxBody: DefaultMaxBodyBytes,
}
}
func defaultLimits() ServerLimits {
return ServerLimits{
MaxBodyBytes: DefaultMaxBodyBytes,
PingSeconds: DefaultPingSeconds,
ReconnectMinSeconds: DefaultReconnectMin,
ReconnectMaxSeconds: DefaultReconnectMax,
JobTimeoutSeconds: DefaultJobTimeout,
MaxFrameBytes: MaxInboundMessage,
}
}
func (s *Session) Backoff(attempt int) time.Duration {
low := s.limits.ReconnectMinSeconds
high := s.limits.ReconnectMaxSeconds
if low < 1 {
low = DefaultReconnectMin
}
if high < low {
high = low
}
if attempt > 16 {
attempt = 16
}
ceiling := low << attempt
if ceiling > high || ceiling <= 0 {
ceiling = high
}
seconds := float64(low) + s.jitter()*float64(ceiling-low)
return time.Duration(seconds * float64(time.Second))
}
func sleepContext(ctx context.Context, d time.Duration) {
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
case <-timer.C:
}
}
func (s *Session) Run(ctx context.Context) error {
attempt := 0
for ctx.Err() == nil {
result := s.connectOnce(ctx)
if result.fatal != "" {
return &FatalError{Reason: result.fatal}
}
if ctx.Err() != nil {
break
}
if result.stable {
attempt = 0
}
var delay time.Duration
if result.fast {
delay = time.Duration((0.5 + s.jitter()) * float64(FastRetryMillis) * float64(time.Millisecond))
} else {
delay = s.Backoff(attempt)
attempt++
}
s.log.Printf("disconnected (%s), reconnecting in %s", result.reason, delay.Round(100*time.Millisecond))
sleepContext(ctx, delay)
}
return nil
}
type ticketAnswer struct {
Ticket string `json:"ticket"`
WSPath string `json:"ws_path"`
}
type errorAnswer struct {
Error struct {
Message string `json:"message"`
Reason string `json:"reason"`
} `json:"error"`
}
func (s *Session) fetchTicket(ctx context.Context) (ticketAnswer, outcome) {
var answer ticketAnswer
request, err := http.NewRequestWithContext(ctx, http.MethodPost, s.cfg.BaseURL()+"/nodes/ticket", nil)
if err != nil {
return answer, outcome{fatal: "bad server url"}
}
request.Header.Set("Authorization", "Bearer "+s.cfg.Secret)
request.Header.Set("Accept", "application/json")
request.Header.Set("User-Agent", "devplace-node/"+Version)
response, err := s.httpc.Do(request)
if err != nil {
return answer, outcome{reason: "ticket request failed: " + stripURL(err).Error()}
}
defer response.Body.Close()
body, _ := io.ReadAll(io.LimitReader(response.Body, 64*1024))
switch {
case response.StatusCode == http.StatusOK:
if json.Unmarshal(body, &answer) != nil || answer.Ticket == "" || !strings.HasPrefix(answer.WSPath, "/") {
return answer, outcome{reason: "ticket answer is malformed"}
}
return answer, outcome{}
case response.StatusCode == http.StatusUnauthorized:
return answer, outcome{fatal: "unauthorized, the device secret is not valid (revoked or rotated)"}
case response.StatusCode == http.StatusForbidden:
var failure errorAnswer
_ = json.Unmarshal(body, &failure)
reason := failure.Error.Reason
if FatalReasons[reason] {
return answer, outcome{fatal: reason}
}
return answer, outcome{reason: "ticket refused: " + reason}
default:
return answer, outcome{reason: fmt.Sprintf("ticket request answered http %d", response.StatusCode)}
}
}
func (s *Session) socketURL(answer ticketAnswer) (string, error) {
base, err := url.Parse(s.cfg.BaseURL())
if err != nil {
return "", err
}
scheme := "ws"
if base.Scheme == "https" {
scheme = "wss"
}
return fmt.Sprintf("%s://%s%s?ticket=%s", scheme, base.Host, answer.WSPath, url.QueryEscape(answer.Ticket)), nil
}
func (s *Session) connectOnce(ctx context.Context) outcome {
answer, failed := s.fetchTicket(ctx)
if failed.fatal != "" || failed.reason != "" {
return failed
}
target, err := s.socketURL(answer)
if err != nil {
return outcome{fatal: "bad server url"}
}
conn, err := s.dialWS(ctx, target)
if err != nil {
return outcome{reason: err.Error()}
}
defer conn.Close()
sessionCtx, cancel := context.WithCancel(ctx)
defer cancel()
go func() {
<-sessionCtx.Done()
conn.Close()
}()
started := time.Now()
result := s.converse(sessionCtx, conn)
if time.Since(started) >= time.Duration(StableSessionSeconds)*time.Second {
result.stable = true
}
return result
}
func (s *Session) sender(conn *Conn) Sender {
return func(frameType string, fields map[string]any) error {
payload := map[string]any{"type": frameType}
for key, value := range fields {
payload[key] = value
}
data, err := json.Marshal(payload)
if err != nil {
return err
}
limit := MaxInboundMessage
if frameType == "job_result" {
limit = MaxResultMessage
}
if len(data) > limit {
return fmt.Errorf("%s frame of %d bytes exceeds the %d byte cap", frameType, len(data), limit)
}
return conn.WriteText(string(data))
}
}
func (s *Session) helloFields() map[string]any {
return map[string]any{
"protocol": ProtocolVersion,
"client_version": Version,
"kind": DeviceKind,
"platform": runtime.GOOS + "/" + runtime.GOARCH,
"modes": []string{"fetch", "tunnel"},
"limits": map[string]any{"max_concurrent": s.limiter.ConcurrencyCap()},
}
}
func (s *Session) applyWelcome(frame Frame) {
limits := defaultLimits()
if frame.Limits != nil {
given := *frame.Limits
if given.MaxBodyBytes > 0 {
limits.MaxBodyBytes = given.MaxBodyBytes
}
if given.PingSeconds > 0 {
limits.PingSeconds = given.PingSeconds
}
if given.ReconnectMinSeconds > 0 {
limits.ReconnectMinSeconds = given.ReconnectMinSeconds
}
if given.ReconnectMaxSeconds > 0 {
limits.ReconnectMaxSeconds = given.ReconnectMaxSeconds
}
limits.MaxConcurrentPerNode = given.MaxConcurrentPerNode
limits.MaxDonorMBPerDay = given.MaxDonorMBPerDay
if given.JobTimeoutSeconds > 0 {
limits.JobTimeoutSeconds = given.JobTimeoutSeconds
}
}
s.limits = limits
s.maxBody = min(limits.MaxBodyBytes, DefaultMaxBodyBytes)
if rejected := s.policy.Refine(frame.Destinations); len(rejected) > 0 {
s.log.Printf("server offered destinations outside the baked-in list, ignored: %s", strings.Join(rejected, ", "))
}
s.limiter.ApplyServer(&limits, frame.Settings)
}
func (s *Session) converse(ctx context.Context, conn *Conn) outcome {
send := s.sender(conn)
if err := send("hello", s.helloFields()); err != nil {
return outcome{reason: "hello failed: " + err.Error()}
}
_ = conn.SetReadDeadline(time.Now().Add(time.Duration(HelloTimeoutSeconds) * time.Second))
raw, err := conn.ReadMessage()
if err != nil {
return s.closeOutcome(err)
}
welcome, err := decodeFrame(raw)
if err != nil {
return s.protocolFailure(conn, "welcome is not valid json")
}
if welcome.Type == "kick" {
return s.kickOutcome(welcome.Reason)
}
if welcome.Type != "welcome" {
return s.protocolFailure(conn, "expected welcome, got "+welcome.Type)
}
s.applyWelcome(welcome)
s.log.Debugf("connected as device %s with %d allowed destinations", welcome.DeviceUID, len(s.policy.Destinations()))
var jobs sync.WaitGroup
tunnels := NewTunnels(s.policy, s.limiter, s.stats, send, s.cfg.MaxTunnels)
background, stop := context.WithCancel(ctx)
jobs.Add(1)
go func() {
defer jobs.Done()
s.housekeeping(background, send)
}()
defer func() {
stop()
tunnels.CloseAll()
jobs.Wait()
s.limiter.Flush()
}()
patience := time.Duration(s.limits.PingSeconds*MissedPingsBeforeDead) * time.Second
for ctx.Err() == nil {
_ = conn.SetReadDeadline(time.Now().Add(patience))
raw, err := conn.ReadMessage()
if err != nil {
return s.closeOutcome(err)
}
frame, err := decodeFrame(raw)
if err != nil {
return s.protocolFailure(conn, "frame is not valid json")
}
switch frame.Type {
case "ping":
if err := send("pong", nil); err != nil {
return outcome{reason: "pong failed: " + err.Error()}
}
case "job":
s.startJob(background, &jobs, send, frame)
case "tunnel_open":
tunnels.Open(background, frame)
case "tunnel_data":
tunnels.Data(frame)
case "tunnel_close":
tunnels.Close(frame.ConnID)
case "settings_update":
s.limiter.ApplyServer(nil, frame.Settings)
case "kick":
return s.kickOutcome(frame.Reason)
default:
return s.protocolFailure(conn, "unexpected frame "+frame.Type)
}
}
return outcome{reason: "stopped"}
}
func (s *Session) protocolFailure(conn *Conn, reason string) outcome {
_ = conn.WriteClose(CloseProtocol, "protocol")
return outcome{reason: "protocol error: " + reason}
}
func (s *Session) kickOutcome(reason string) outcome {
if FatalReasons[reason] {
return outcome{fatal: reason}
}
return outcome{reason: "kicked: " + reason}
}
func (s *Session) closeOutcome(err error) outcome {
var closeErr *CloseError
if !errors.As(err, &closeErr) {
return outcome{reason: "connection lost: " + err.Error()}
}
if FatalReasons[closeErr.Reason] {
return outcome{fatal: closeErr.Reason}
}
if closeErr.Code == CloseRetryOwner {
return outcome{fast: true, reason: "server asked for a retry on the owner worker"}
}
return outcome{reason: closeErr.Error()}
}
func (s *Session) housekeeping(ctx context.Context, send Sender) {
ticker := time.NewTicker(time.Duration(UsageIntervalSeconds) * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if line := s.stats.Roll(time.Now()); line != "" {
s.log.Printf("%s", line)
}
up, down := s.stats.Take()
if up+down > 0 {
_ = send("usage", map[string]any{"bytes_up": up, "bytes_down": down})
}
}
}
}
func (s *Session) startJob(ctx context.Context, jobs *sync.WaitGroup, send Sender, frame Frame) {
if frame.JobID == "" || len(frame.JobID) > MaxIDBytes {
s.log.Printf("job without a usable id ignored")
return
}
if code := s.limiter.Acquire(); code != "" {
s.stats.Refused()
_ = send("job_error", map[string]any{"job_id": frame.JobID, "code": code})
return
}
jobs.Add(1)
go func() {
defer jobs.Done()
defer s.limiter.Release()
s.runJob(ctx, send, frame)
}()
}
func (s *Session) runJob(ctx context.Context, send Sender, frame Frame) {
result, err := s.fetcher.Do(ctx, frame, s.maxBody, s.limiter.Remaining())
if err != nil {
code := ErrorCode(err)
var policyErr *PolicyError
if errors.As(err, &policyErr) {
s.stats.Refused()
} else {
s.stats.Failed()
}
s.log.Debugf("job %s failed: %s", frame.JobID, code)
_ = send("job_error", map[string]any{"job_id": frame.JobID, "code": code})
return
}
up := int64(len(frame.URL) + 256)
down := int64(len(result.Body))
s.limiter.AddBytes(up + down)
s.stats.Job(up, down)
fields := map[string]any{
"job_id": frame.JobID,
"status": result.Status,
"headers": result.Headers,
"body_b64": base64.StdEncoding.EncodeToString(result.Body),
"truncated": result.Truncated,
"elapsed_ms": int(result.Elapsed.Milliseconds()),
}
if err := send("job_result", fields); err != nil {
s.log.Printf("job %s result could not be sent: %v", frame.JobID, err)
}
}
nodes/client/session_test.go
+362
View File
@@ -0,0 +1,362 @@
package main
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"net"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
)
const testSecret = "dpn_testsecret0123456789"
type harness struct {
server *httptest.Server
content *httptest.Server
session *Session
policy *Policy
conns atomic.Int32
tickets atomic.Int32
failure atomic.Value
}
func readFrame(t *testing.T, c *Conn) map[string]any {
t.Helper()
_ = c.SetReadDeadline(time.Now().Add(10 * time.Second))
raw, err := c.ReadMessage()
if err != nil {
t.Errorf("fake server read: %v", err)
return map[string]any{"type": "read_error"}
}
frame := map[string]any{}
if err := json.Unmarshal(raw, &frame); err != nil {
t.Errorf("fake server decode: %v", err)
}
return frame
}
func writeFrame(t *testing.T, c *Conn, frame map[string]any) {
t.Helper()
data, _ := json.Marshal(frame)
if err := c.WriteText(string(data)); err != nil {
t.Errorf("fake server write: %v", err)
}
}
func welcome() map[string]any {
return map[string]any{
"type": "welcome",
"device_uid": "dev1",
"limits": map[string]any{
"max_body_bytes": 1000, "ping_seconds": 20, "reconnect_min_seconds": 1, "reconnect_max_seconds": 2,
"max_concurrent_per_node": 2, "max_donor_mb_per_day": 20480, "job_timeout_seconds": 25,
},
"destinations": []any{
map[string]any{"suffix": "reddit.com", "adapter": "reddit", "methods": []string{"GET"}, "ports": []int{80, 443}, "modes": []string{"tunnel", "fetch"}, "max_rps": 0.5},
map[string]any{"suffix": "redd.it", "adapter": "reddit", "methods": []string{"GET"}, "ports": []int{80, 443}, "modes": []string{"tunnel", "fetch"}, "max_rps": 0.5},
},
"settings": map[string]any{"enabled": true, "bandwidth_mb_day": 500, "max_concurrent": 2, "active_from": "", "active_to": "", "timezone": "UTC"},
}
}
func newHarness(t *testing.T, script func(n int, c *Conn)) *harness {
t.Helper()
h := &harness{}
h.content = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Cookie") != "" || r.Header.Get("Authorization") != "" {
h.failure.Store("identity header reached the destination")
}
_, _ = w.Write([]byte(strings.Repeat("r", 5000)))
}))
mux := http.NewServeMux()
mux.HandleFunc("/nodes/ticket", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.Header.Get("Authorization") != "Bearer "+testSecret {
w.WriteHeader(http.StatusUnauthorized)
return
}
n := h.tickets.Add(1)
_ = json.NewEncoder(w).Encode(map[string]any{"ticket": "t" + string(rune('0'+n)), "expires_in": 30, "ws_path": "/nodes/ws"})
})
mux.HandleFunc("/nodes/ws", func(w http.ResponseWriter, r *http.Request) {
if !strings.HasPrefix(r.URL.Query().Get("ticket"), "t") || strings.Contains(r.URL.RawQuery, testSecret) {
h.failure.Store("bad ticket query")
}
conn, err := upgrade(w, r)
if err != nil {
return
}
defer conn.Close()
script(int(h.conns.Add(1)), conn)
})
h.server = httptest.NewServer(mux)
t.Cleanup(h.server.Close)
t.Cleanup(h.content.Close)
cfg := DefaultConfig()
cfg.Server = h.server.URL
cfg.Secret = testSecret
cfg.Timezone = "UTC"
h.policy = testPolicy(publicAnswer)
h.policy.dial = func(ctx context.Context, network, address string) (net.Conn, error) {
var dialer net.Dialer
return dialer.DialContext(ctx, "tcp", h.content.Listener.Addr().String())
}
limiter, err := NewLimiter(cfg, "", time.Now)
if err != nil {
t.Fatal(err)
}
h.session = NewSession(cfg, h.policy, limiter, NewFetcher(h.policy), NewStats(time.Now()), NewLogger(testWriter{t}, true))
h.session.jitter = func() float64 { return 0 }
return h
}
type testWriter struct{ t *testing.T }
func (w testWriter) Write(p []byte) (int, error) {
w.t.Log(strings.TrimSpace(string(p)))
return len(p), nil
}
func (h *harness) run(t *testing.T) error {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
err := h.session.Run(ctx)
if failure := h.failure.Load(); failure != nil {
t.Error(failure)
}
return err
}
func TestSessionServesJobsRefusesBadOnesAndStopsOnKick(t *testing.T) {
h := newHarness(t, func(n int, c *Conn) {
hello := readFrame(t, c)
if hello["type"] != "hello" || hello["protocol"] != float64(1) || hello["kind"] != "native" || hello["client_version"] != Version {
t.Errorf("hello %v", hello)
}
if modes, _ := hello["modes"].([]any); len(modes) == 0 {
t.Errorf("hello modes %v", hello["modes"])
}
if n == 2 {
writeFrame(t, c, welcome())
writeFrame(t, c, map[string]any{"type": "kick", "reason": "revoked"})
return
}
writeFrame(t, c, welcome())
writeFrame(t, c, map[string]any{"type": "ping"})
if frame := readFrame(t, c); frame["type"] != "pong" {
t.Errorf("expected pong, got %v", frame)
}
writeFrame(t, c, map[string]any{"type": "job", "job_id": "ok1", "method": "GET", "url": "http://www.reddit.com/r/go.json", "timeout": 5})
result := readFrame(t, c)
if result["type"] != "job_result" || result["job_id"] != "ok1" || result["status"] != float64(200) || result["truncated"] != true {
t.Errorf("result %v", result)
} else {
body, err := base64.StdEncoding.DecodeString(result["body_b64"].(string))
if err != nil || len(body) != 1000 {
t.Errorf("body %d %v", len(body), err)
}
}
refused := map[string]string{
"bad1": "http://evilreddit.com/", "bad2": "http://reddit.com:8080/", "bad3": "http://127.0.0.1/",
}
for id, target := range refused {
writeFrame(t, c, map[string]any{"type": "job", "job_id": id, "method": "GET", "url": target, "timeout": 5})
answer := readFrame(t, c)
want := RefuseHost
if id == "bad2" {
want = RefusePort
}
if answer["type"] != "job_error" || answer["job_id"] != id || answer["code"] != want {
t.Errorf("%s: %v", id, answer)
}
}
writeFrame(t, c, map[string]any{"type": "job", "job_id": "post", "method": "POST", "url": "http://reddit.com/", "timeout": 5})
if answer := readFrame(t, c); answer["code"] != RefuseMethod {
t.Errorf("post: %v", answer)
}
writeFrame(t, c, map[string]any{"type": "tunnel_open", "conn_id": "c1", "host": "evil.com", "port": 443})
if answer := readFrame(t, c); answer["type"] != "tunnel_opened" || answer["ok"] != false || answer["code"] != RefuseHost {
t.Errorf("tunnel: %v", answer)
}
_ = c.WriteClose(CloseRetryOwner, "owner")
})
started := time.Now()
err := h.run(t)
var fatal *FatalError
if !errors.As(err, &fatal) || fatal.Reason != "revoked" {
t.Fatalf("run returned %v", err)
}
if h.conns.Load() != 2 || h.tickets.Load() != 2 {
t.Fatalf("connections %d tickets %d", h.conns.Load(), h.tickets.Load())
}
if time.Since(started) > 5*time.Second {
t.Fatal("4013 was not retried quickly")
}
if line := h.session.stats.Line(); !strings.Contains(line, "jobs=1") || !strings.Contains(line, "refused=") {
t.Fatalf("stats %s", line)
}
}
func TestSessionRelaysTunnelBothWays(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer listener.Close()
go func() {
conn, err := listener.Accept()
if err != nil {
return
}
defer conn.Close()
buffer := make([]byte, 64)
for {
n, err := conn.Read(buffer)
if err != nil {
return
}
_, _ = conn.Write([]byte("echo:" + string(buffer[:n])))
}
}()
h := newHarness(t, func(n int, c *Conn) {
readFrame(t, c)
writeFrame(t, c, welcome())
if n == 2 {
writeFrame(t, c, map[string]any{"type": "kick", "reason": "kill_switch"})
return
}
writeFrame(t, c, map[string]any{"type": "tunnel_open", "conn_id": "c1", "host": "www.reddit.com", "port": 443})
if answer := readFrame(t, c); answer["type"] != "tunnel_opened" || answer["ok"] != true {
t.Errorf("opened: %v", answer)
return
}
writeFrame(t, c, map[string]any{"type": "tunnel_data", "conn_id": "c1", "b64": base64.StdEncoding.EncodeToString([]byte("hi"))})
data := readFrame(t, c)
decoded, _ := base64.StdEncoding.DecodeString(data["b64"].(string))
if data["type"] != "tunnel_data" || string(decoded) != "echo:hi" {
t.Errorf("data: %v %q", data, decoded)
}
writeFrame(t, c, map[string]any{"type": "tunnel_close", "conn_id": "c1"})
_ = c.WriteClose(CloseGoingAway, "restart")
})
h.policy.dial = func(ctx context.Context, network, address string) (net.Conn, error) {
var dialer net.Dialer
return dialer.DialContext(ctx, "tcp", listener.Addr().String())
}
err = h.run(t)
var fatal *FatalError
if !errors.As(err, &fatal) || fatal.Reason != "kill_switch" {
t.Fatalf("run returned %v", err)
}
}
func TestSessionLimitsAnswerWithDonorCodes(t *testing.T) {
h := newHarness(t, func(n int, c *Conn) {
readFrame(t, c)
writeFrame(t, c, welcome())
if n == 1 {
writeFrame(t, c, map[string]any{"type": "job", "job_id": "a", "method": "GET", "url": "http://reddit.com/", "timeout": 5})
if answer := readFrame(t, c); answer["code"] != RefuseBandwidth {
t.Errorf("bandwidth: %v", answer)
}
_ = c.WriteClose(CloseGoingAway, "bye")
return
}
writeFrame(t, c, map[string]any{"type": "kick", "reason": "suspended"})
})
h.session.limiter.AddBytes(1 << 40)
err := h.run(t)
var fatal *FatalError
if !errors.As(err, &fatal) || fatal.Reason != "suspended" {
t.Fatalf("run returned %v", err)
}
}
func TestFatalTicketStatuses(t *testing.T) {
cases := []struct {
status int
body string
fatal string
}{
{401, "", "unauthorized"},
{403, `{"error":{"status":403,"message":"x","reason":"no_consent"}}`, "no_consent"},
{403, `{"error":{"status":403,"message":"x","reason":"suspended"}}`, "suspended"},
}
for _, item := range cases {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(item.status)
_, _ = w.Write([]byte(item.body))
}))
cfg := DefaultConfig()
cfg.Server, cfg.Secret = server.URL, testSecret
limiter, _ := NewLimiter(cfg, "", time.Now)
policy := testPolicy(publicAnswer)
session := NewSession(cfg, policy, limiter, NewFetcher(policy), NewStats(time.Now()), NewLogger(testWriter{t}, true))
err := session.Run(context.Background())
var fatal *FatalError
if !errors.As(err, &fatal) || !strings.Contains(fatal.Reason, item.fatal) {
t.Errorf("%d: %v", item.status, err)
}
server.Close()
}
}
func TestTransientTicketFailureRetries(t *testing.T) {
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if calls.Add(1) < 3 {
w.WriteHeader(http.StatusBadGateway)
return
}
w.WriteHeader(http.StatusUnauthorized)
}))
defer server.Close()
cfg := DefaultConfig()
cfg.Server, cfg.Secret = server.URL, testSecret
limiter, _ := NewLimiter(cfg, "", time.Now)
policy := testPolicy(publicAnswer)
session := NewSession(cfg, policy, limiter, NewFetcher(policy), NewStats(time.Now()), NewLogger(testWriter{t}, true))
session.jitter = func() float64 { return 0 }
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
var fatal *FatalError
if err := session.Run(ctx); !errors.As(err, &fatal) || calls.Load() != 3 {
t.Fatalf("err %v calls %d", err, calls.Load())
}
}
func TestBackoffStaysInsideBounds(t *testing.T) {
session := &Session{limits: ServerLimits{ReconnectMinSeconds: 1, ReconnectMaxSeconds: 60}}
for _, jitter := range []float64{0, 0.5, 0.999999} {
session.jitter = func() float64 { return jitter }
for attempt := 0; attempt < 40; attempt++ {
d := session.Backoff(attempt)
if d < time.Second || d > 60*time.Second {
t.Fatalf("attempt %d jitter %v: %v", attempt, jitter, d)
}
}
}
session.jitter = func() float64 { return 0.999999 }
if first, later := session.Backoff(0), session.Backoff(5); first > time.Second+time.Millisecond || later <= first {
t.Fatalf("growth: %v %v", first, later)
}
}
func TestFatalReasonsMatchPlan(t *testing.T) {
for _, reason := range []string{"revoked", "suspended", "kill_switch", "no_consent"} {
if !FatalReasons[reason] {
t.Errorf("%s must stop reconnecting", reason)
}
}
for _, reason := range []string{"heartbeat", "ticket", "capacity"} {
if FatalReasons[reason] {
t.Errorf("%s must reconnect", reason)
}
}
}
nodes/client/stats.go
+90
View File
@@ -0,0 +1,90 @@
package main
import (
"fmt"
"sync"
"time"
)
type Stats struct {
mu sync.Mutex
day string
jobs int
refused int
failed int
tunnels int
up int64
down int64
sentUp int64
sentDown int64
}
func NewStats(now time.Time) *Stats {
return &Stats{day: now.Format("2006-01-02")}
}
func (s *Stats) Job(up, down int64) {
s.mu.Lock()
s.jobs++
s.up += up
s.down += down
s.mu.Unlock()
}
func (s *Stats) Refused() {
s.mu.Lock()
s.refused++
s.mu.Unlock()
}
func (s *Stats) Failed() {
s.mu.Lock()
s.failed++
s.mu.Unlock()
}
func (s *Stats) Tunnel() {
s.mu.Lock()
s.tunnels++
s.mu.Unlock()
}
func (s *Stats) Bytes(up, down int64) {
s.mu.Lock()
s.up += up
s.down += down
s.mu.Unlock()
}
func (s *Stats) Take() (up, down int64) {
s.mu.Lock()
defer s.mu.Unlock()
up, down = s.up-s.sentUp, s.down-s.sentDown
s.sentUp, s.sentDown = s.up, s.down
return
}
func (s *Stats) line() string {
return fmt.Sprintf("totals day=%s jobs=%d refused=%d failed=%d tunnels=%d bytes_up=%d bytes_down=%d",
s.day, s.jobs, s.refused, s.failed, s.tunnels, s.up, s.down)
}
func (s *Stats) Line() string {
s.mu.Lock()
defer s.mu.Unlock()
return s.line()
}
func (s *Stats) Roll(now time.Time) string {
s.mu.Lock()
defer s.mu.Unlock()
today := now.Format("2006-01-02")
if today == s.day {
return ""
}
line := s.line()
s.day = today
s.jobs, s.refused, s.failed, s.tunnels = 0, 0, 0, 0
s.up, s.down, s.sentUp, s.sentDown = 0, 0, 0, 0
return line
}
nodes/client/tunnel.go
+201
View File
@@ -0,0 +1,201 @@
package main
import (
"context"
"encoding/base64"
"net"
"strconv"
"sync"
"time"
)
type Sender func(frameType string, fields map[string]any) error
type tunnel struct {
cancel context.CancelFunc
out chan []byte
remote bool
}
type Tunnels struct {
policy *Policy
limiter *Limiter
stats *Stats
send Sender
max int
mu sync.Mutex
items map[string]*tunnel
wg sync.WaitGroup
}
func NewTunnels(policy *Policy, limiter *Limiter, stats *Stats, send Sender, max int) *Tunnels {
return &Tunnels{policy: policy, limiter: limiter, stats: stats, send: send, max: max, items: map[string]*tunnel{}}
}
func (t *Tunnels) deny(id, code string) {
_ = t.send("tunnel_opened", map[string]any{"conn_id": id, "ok": false, "code": code})
}
func (t *Tunnels) Open(ctx context.Context, frame Frame) {
id := frame.ConnID
if id == "" || len(id) > MaxIDBytes {
return
}
if _, err := t.policy.CheckHostPort(frame.Host, frame.Port, "tunnel"); err != nil {
t.deny(id, ErrorCode(err))
t.stats.Refused()
return
}
t.mu.Lock()
_, exists := t.items[id]
full := len(t.items) >= t.max
t.mu.Unlock()
if exists || full {
t.deny(id, RefuseConcurrency)
t.stats.Refused()
return
}
if code := t.limiter.Acquire(); code != "" {
t.deny(id, code)
t.stats.Refused()
return
}
runCtx, cancel := context.WithTimeout(ctx, time.Duration(TunnelLifetimeSeconds)*time.Second)
item := &tunnel{cancel: cancel, out: make(chan []byte, TunnelQueueDepth)}
t.mu.Lock()
if _, taken := t.items[id]; taken || len(t.items) >= t.max {
t.mu.Unlock()
cancel()
t.limiter.Release()
t.deny(id, RefuseConcurrency)
t.stats.Refused()
return
}
t.items[id] = item
t.mu.Unlock()
t.wg.Add(1)
go t.run(runCtx, id, frame.Host, frame.Port, item)
}
func (t *Tunnels) finish(id string, item *tunnel, announce bool) {
t.mu.Lock()
if t.items[id] == item {
delete(t.items, id)
}
t.mu.Unlock()
item.cancel()
t.limiter.Release()
if announce {
_ = t.send("tunnel_close", map[string]any{"conn_id": id})
}
}
func (t *Tunnels) run(ctx context.Context, id, host string, port int, item *tunnel) {
defer t.wg.Done()
conn, err := t.policy.DialContext(ctx, "tcp", net.JoinHostPort(host, strconv.Itoa(port)))
if err != nil {
t.finish(id, item, false)
t.deny(id, ErrorCode(err))
t.stats.Failed()
return
}
t.stats.Tunnel()
if err := t.send("tunnel_opened", map[string]any{"conn_id": id, "ok": true}); err != nil {
conn.Close()
t.finish(id, item, false)
return
}
go func() {
<-ctx.Done()
conn.Close()
}()
var writers sync.WaitGroup
writers.Add(1)
go func() {
defer writers.Done()
for {
select {
case <-ctx.Done():
return
case chunk := <-item.out:
_ = conn.SetWriteDeadline(time.Now().Add(10 * time.Second))
if _, err := conn.Write(chunk); err != nil {
item.cancel()
return
}
t.stats.Bytes(int64(len(chunk)), 0)
t.limiter.AddBytes(int64(len(chunk)))
}
}
}()
buffer := make([]byte, TunnelChunkBytes)
for ctx.Err() == nil {
_ = conn.SetReadDeadline(time.Now().Add(time.Duration(TunnelIdleSeconds) * time.Second))
n, err := conn.Read(buffer)
if n > 0 {
t.stats.Bytes(0, int64(n))
t.limiter.AddBytes(int64(n))
encoded := base64.StdEncoding.EncodeToString(buffer[:n])
if sendErr := t.send("tunnel_data", map[string]any{"conn_id": id, "b64": encoded}); sendErr != nil {
break
}
if t.limiter.Remaining() == 0 {
break
}
}
if err != nil {
break
}
}
conn.Close()
item.cancel()
writers.Wait()
t.mu.Lock()
announce := !item.remote
t.mu.Unlock()
t.finish(id, item, announce)
}
func (t *Tunnels) Data(frame Frame) {
t.mu.Lock()
item := t.items[frame.ConnID]
t.mu.Unlock()
if item == nil {
return
}
if len(frame.B64) > MaxInboundMessage {
item.cancel()
return
}
chunk, err := base64.StdEncoding.Strict().DecodeString(frame.B64)
if err != nil {
item.cancel()
return
}
select {
case item.out <- chunk:
default:
item.cancel()
}
}
func (t *Tunnels) Close(id string) {
t.mu.Lock()
item := t.items[id]
if item != nil {
item.remote = true
}
t.mu.Unlock()
if item != nil {
item.cancel()
}
}
func (t *Tunnels) CloseAll() {
t.mu.Lock()
for _, item := range t.items {
item.cancel()
}
t.mu.Unlock()
t.wg.Wait()
}
nodes/client/ws.go
+358
View File
@@ -0,0 +1,358 @@
package main
import (
"bufio"
"context"
"crypto/rand"
"crypto/sha1"
"crypto/tls"
"encoding/base64"
"encoding/binary"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"strings"
"sync"
"time"
"unicode/utf8"
)
const (
opContinuation = 0x0
opText = 0x1
opBinary = 0x2
opClose = 0x8
opPing = 0x9
opPong = 0xA
wsGUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
wsWriteTimeout = 30 * time.Second
wsHandshakeLimit = 15 * time.Second
maxControlBytes = 125
)
type CloseError struct {
Code int
Reason string
}
func (e *CloseError) Error() string {
return fmt.Sprintf("websocket closed code=%d reason=%q", e.Code, e.Reason)
}
type Conn struct {
rwc net.Conn
br *bufio.Reader
client bool
maxMessage int
wmu sync.Mutex
closed bool
}
func newConn(rwc net.Conn, br *bufio.Reader, client bool) *Conn {
return &Conn{rwc: rwc, br: br, client: client, maxMessage: MaxInboundMessage}
}
func Dial(ctx context.Context, rawURL string) (*Conn, error) {
target, err := url.Parse(rawURL)
if err != nil {
return nil, errors.New("websocket url cannot be parsed")
}
secure := false
switch target.Scheme {
case "wss", "https":
secure = true
case "ws", "http":
default:
return nil, errors.New("websocket url scheme must be ws, wss, http or https")
}
host := target.Hostname()
port := target.Port()
if port == "" {
port = "80"
if secure {
port = "443"
}
}
address := net.JoinHostPort(host, port)
dialer := &net.Dialer{Timeout: wsHandshakeLimit}
raw, err := dialer.DialContext(ctx, "tcp", address)
if err != nil {
return nil, fmt.Errorf("websocket connect to %s failed: %w", address, stripURL(err))
}
var rwc net.Conn = raw
if secure {
tlsConn := tls.Client(raw, &tls.Config{ServerName: host, MinVersion: tls.VersionTLS12})
_ = raw.SetDeadline(time.Now().Add(wsHandshakeLimit))
if err := tlsConn.HandshakeContext(ctx); err != nil {
raw.Close()
return nil, fmt.Errorf("websocket tls handshake failed: %w", err)
}
rwc = tlsConn
}
_ = rwc.SetDeadline(time.Now().Add(wsHandshakeLimit))
keyBytes := make([]byte, 16)
if _, err := rand.Read(keyBytes); err != nil {
rwc.Close()
return nil, err
}
key := base64.StdEncoding.EncodeToString(keyBytes)
path := target.RequestURI()
if path == "" {
path = "/"
}
request := fmt.Sprintf(
"GET %s HTTP/1.1\r\nHost: %s\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: %s\r\nSec-WebSocket-Version: 13\r\nUser-Agent: devplace-node/%s\r\n\r\n",
path, target.Host, key, Version,
)
if _, err := io.WriteString(rwc, request); err != nil {
rwc.Close()
return nil, fmt.Errorf("websocket handshake write failed: %w", err)
}
br := bufio.NewReaderSize(rwc, 16*1024)
response, err := http.ReadResponse(br, &http.Request{Method: "GET"})
if err != nil {
rwc.Close()
return nil, fmt.Errorf("websocket handshake read failed: %w", err)
}
if response.StatusCode != http.StatusSwitchingProtocols {
rwc.Close()
return nil, fmt.Errorf("websocket handshake refused with http status %d", response.StatusCode)
}
if !strings.EqualFold(response.Header.Get("Upgrade"), "websocket") ||
response.Header.Get("Sec-WebSocket-Accept") != acceptKey(key) {
rwc.Close()
return nil, errors.New("websocket handshake answer is invalid")
}
_ = rwc.SetDeadline(time.Time{})
return newConn(rwc, br, true), nil
}
func stripURL(err error) error {
var urlErr *url.Error
if errors.As(err, &urlErr) {
return urlErr.Err
}
return err
}
func acceptKey(key string) string {
sum := sha1.Sum([]byte(key + wsGUID))
return base64.StdEncoding.EncodeToString(sum[:])
}
func (c *Conn) SetReadDeadline(t time.Time) error { return c.rwc.SetReadDeadline(t) }
func (c *Conn) Close() error {
c.wmu.Lock()
c.closed = true
c.wmu.Unlock()
return c.rwc.Close()
}
func (c *Conn) writeFrame(fin bool, op byte, payload []byte) error {
header := make([]byte, 0, 14)
first := op
if fin {
first |= 0x80
}
header = append(header, first)
maskBit := byte(0)
if c.client {
maskBit = 0x80
}
size := len(payload)
switch {
case size <= 125:
header = append(header, maskBit|byte(size))
case size <= 0xFFFF:
header = append(header, maskBit|126, 0, 0)
binary.BigEndian.PutUint16(header[len(header)-2:], uint16(size))
default:
header = append(header, maskBit|127, 0, 0, 0, 0, 0, 0, 0, 0)
binary.BigEndian.PutUint64(header[len(header)-8:], uint64(size))
}
body := payload
if c.client {
mask := make([]byte, 4)
if _, err := rand.Read(mask); err != nil {
return err
}
header = append(header, mask...)
body = make([]byte, size)
for i := range payload {
body[i] = payload[i] ^ mask[i%4]
}
}
if err := c.rwc.SetWriteDeadline(time.Now().Add(wsWriteTimeout)); err != nil {
return err
}
if _, err := c.rwc.Write(header); err != nil {
return err
}
_, err := c.rwc.Write(body)
return err
}
func (c *Conn) WriteText(text string) error {
data := []byte(text)
c.wmu.Lock()
defer c.wmu.Unlock()
if c.closed {
return net.ErrClosed
}
op := byte(opText)
for first := true; first || len(data) > 0; first = false {
chunk := data
if len(chunk) > MaxWSFramePayload {
chunk = data[:MaxWSFramePayload]
}
data = data[len(chunk):]
if err := c.writeFrame(len(data) == 0, op, chunk); err != nil {
return err
}
op = opContinuation
}
return nil
}
func (c *Conn) writeControl(op byte, payload []byte) error {
if len(payload) > maxControlBytes {
payload = payload[:maxControlBytes]
}
c.wmu.Lock()
defer c.wmu.Unlock()
if c.closed {
return net.ErrClosed
}
return c.writeFrame(true, op, payload)
}
func (c *Conn) WriteClose(code int, reason string) error {
if len(reason) > 100 {
reason = reason[:100]
}
payload := make([]byte, 2, 2+len(reason))
binary.BigEndian.PutUint16(payload, uint16(code))
payload = append(payload, reason...)
return c.writeControl(opClose, payload)
}
func (c *Conn) readFrame() (fin bool, op byte, payload []byte, err error) {
head := make([]byte, 2)
if _, err = io.ReadFull(c.br, head); err != nil {
return
}
fin = head[0]&0x80 != 0
if head[0]&0x70 != 0 {
err = &CloseError{Code: CloseProtocolError, Reason: "reserved bits set"}
return
}
op = head[0] & 0x0F
masked := head[1]&0x80 != 0
if masked == c.client {
err = &CloseError{Code: CloseProtocolError, Reason: "bad masking"}
return
}
length := uint64(head[1] & 0x7F)
switch length {
case 126:
ext := make([]byte, 2)
if _, err = io.ReadFull(c.br, ext); err != nil {
return
}
length = uint64(binary.BigEndian.Uint16(ext))
case 127:
ext := make([]byte, 8)
if _, err = io.ReadFull(c.br, ext); err != nil {
return
}
length = binary.BigEndian.Uint64(ext)
}
if op&0x8 != 0 && (length > maxControlBytes || !fin) {
err = &CloseError{Code: CloseProtocolError, Reason: "bad control frame"}
return
}
if length > uint64(c.maxMessage) || length > uint64(MaxResultMessage) {
err = &CloseError{Code: CloseTooLarge, Reason: "frame too large"}
return
}
var mask []byte
if masked {
mask = make([]byte, 4)
if _, err = io.ReadFull(c.br, mask); err != nil {
return
}
}
payload = make([]byte, length)
if _, err = io.ReadFull(c.br, payload); err != nil {
return
}
if masked {
for i := range payload {
payload[i] ^= mask[i%4]
}
}
return
}
func (c *Conn) fail(code int, reason string) error {
_ = c.WriteClose(code, reason)
return &CloseError{Code: code, Reason: reason}
}
func (c *Conn) ReadMessage() ([]byte, error) {
var message []byte
inProgress := false
for {
fin, op, payload, err := c.readFrame()
if err != nil {
var closeErr *CloseError
if errors.As(err, &closeErr) {
return nil, c.fail(closeErr.Code, closeErr.Reason)
}
return nil, err
}
switch op {
case opPing:
if err := c.writeControl(opPong, payload); err != nil {
return nil, err
}
case opPong:
case opClose:
code, reason := CloseNormal, ""
if len(payload) >= 2 {
code = int(binary.BigEndian.Uint16(payload))
reason = string(payload[2:])
}
_ = c.WriteClose(code, "")
return nil, &CloseError{Code: code, Reason: reason}
case opText, opBinary, opContinuation:
if op == opContinuation && !inProgress {
return nil, c.fail(CloseProtocolError, "unexpected continuation")
}
if op != opContinuation && inProgress {
return nil, c.fail(CloseProtocolError, "interleaved data frames")
}
if op == opBinary {
return nil, c.fail(CloseUnsupported, "binary frames are not supported")
}
inProgress = !fin
if len(message)+len(payload) > c.maxMessage {
return nil, c.fail(CloseTooLarge, "message too large")
}
message = append(message, payload...)
if fin {
if !utf8.Valid(message) {
return nil, c.fail(CloseInvalidPayload, "invalid utf-8")
}
return message, nil
}
default:
return nil, c.fail(CloseProtocolError, "unknown opcode")
}
}
}
nodes/client/ws_test.go
+220
View File
@@ -0,0 +1,220 @@
package main
import (
"bytes"
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
func upgrade(w http.ResponseWriter, r *http.Request) (*Conn, error) {
hijacker := w.(http.Hijacker)
rwc, brw, err := hijacker.Hijack()
if err != nil {
return nil, err
}
answer := "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: " +
acceptKey(r.Header.Get("Sec-WebSocket-Key")) + "\r\n\r\n"
if _, err := rwc.Write([]byte(answer)); err != nil {
return nil, err
}
conn := newConn(rwc, brw.Reader, false)
conn.maxMessage = MaxResultMessage
return conn, nil
}
func wsServer(t *testing.T, handler func(*Conn)) *httptest.Server {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
conn, err := upgrade(w, r)
if err != nil {
t.Errorf("upgrade: %v", err)
return
}
defer conn.Close()
handler(conn)
}))
t.Cleanup(server.Close)
return server
}
func dialTest(t *testing.T, server *httptest.Server) *Conn {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
conn, err := Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/socket")
if err != nil {
t.Fatalf("dial: %v", err)
}
t.Cleanup(func() { conn.Close() })
_ = conn.SetReadDeadline(time.Now().Add(10 * time.Second))
return conn
}
func TestAcceptKeyVector(t *testing.T) {
if got := acceptKey("dGhlIHNhbXBsZSBub25jZQ=="); got != "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=" {
t.Fatalf("accept key %s", got)
}
}
func TestRoundTripAndFragmentation(t *testing.T) {
sizes := []int{0, 1, 125, 126, 65535, 65536, MaxWSFramePayload, MaxWSFramePayload + 1, 700 * 1024}
server := wsServer(t, func(c *Conn) {
for range sizes {
message, err := c.ReadMessage()
if err != nil {
t.Errorf("server read: %v", err)
return
}
if len(message) <= MaxInboundMessage {
if err := c.WriteText(string(message)); err != nil {
t.Errorf("server write: %v", err)
return
}
} else if err := c.WriteText("big"); err != nil {
return
}
}
})
conn := dialTest(t, server)
for _, size := range sizes {
text := strings.Repeat("a", size)
if err := conn.WriteText(text); err != nil {
t.Fatalf("write %d: %v", size, err)
}
got, err := conn.ReadMessage()
if err != nil {
t.Fatalf("read %d: %v", size, err)
}
want := text
if size > MaxInboundMessage {
want = "big"
}
if string(got) != want {
t.Fatalf("size %d: got %d bytes", size, len(got))
}
}
}
func TestPingAnsweredWithPong(t *testing.T) {
done := make(chan []byte, 1)
server := wsServer(t, func(c *Conn) {
_ = c.writeControl(opPing, []byte("probe"))
_ = c.WriteText("after")
_, op, payload, err := c.readFrame()
if err != nil || op != opPong {
t.Errorf("expected pong, got op %d err %v", op, err)
}
done <- payload
})
conn := dialTest(t, server)
message, err := conn.ReadMessage()
if err != nil || string(message) != "after" {
t.Fatalf("message %q err %v", message, err)
}
if got := <-done; !bytes.Equal(got, []byte("probe")) {
t.Fatalf("pong payload %q", got)
}
}
func TestClientFramesAreMasked(t *testing.T) {
masked := make(chan bool, 1)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
conn, err := upgrade(w, r)
if err != nil {
return
}
defer conn.Close()
head := make([]byte, 2)
if _, err := conn.br.Read(head); err != nil {
return
}
masked <- head[1]&0x80 != 0
}))
defer server.Close()
conn := dialTest(t, server)
if err := conn.WriteText("x"); err != nil {
t.Fatal(err)
}
if !<-masked {
t.Fatal("client frame was not masked")
}
}
func TestCloseCodeSurfaces(t *testing.T) {
server := wsServer(t, func(c *Conn) { _ = c.WriteClose(CloseRetryOwner, "owner") })
conn := dialTest(t, server)
_, err := conn.ReadMessage()
var closeErr *CloseError
if !errors.As(err, &closeErr) || closeErr.Code != CloseRetryOwner || closeErr.Reason != "owner" {
t.Fatalf("got %v", err)
}
}
func TestOversizeFrameRejectedWithClose(t *testing.T) {
server := wsServer(t, func(c *Conn) {
_ = c.writeFrame(true, opText, []byte(strings.Repeat("b", MaxWSFramePayload+1)))
_, _, _, _ = c.readFrame()
})
conn := dialTest(t, server)
_, err := conn.ReadMessage()
var closeErr *CloseError
if !errors.As(err, &closeErr) || closeErr.Code != CloseTooLarge {
t.Fatalf("got %v", err)
}
}
func TestOversizeMessageAcrossFragmentsRejected(t *testing.T) {
server := wsServer(t, func(c *Conn) {
chunk := []byte(strings.Repeat("c", MaxWSFramePayload))
_ = c.writeFrame(false, opText, chunk)
_ = c.writeFrame(true, opContinuation, chunk)
_, _, _, _ = c.readFrame()
})
conn := dialTest(t, server)
_, err := conn.ReadMessage()
var closeErr *CloseError
if !errors.As(err, &closeErr) || closeErr.Code != CloseTooLarge {
t.Fatalf("got %v", err)
}
}
func TestBinaryAndBadUTF8Rejected(t *testing.T) {
cases := []struct {
name string
op byte
payload []byte
code int
}{
{"binary", opBinary, []byte("x"), CloseUnsupported},
{"utf8", opText, []byte{0xff, 0xfe}, CloseInvalidPayload},
{"continuation", opContinuation, []byte("x"), CloseProtocolError},
}
for _, item := range cases {
server := wsServer(t, func(c *Conn) {
_ = c.writeFrame(true, item.op, item.payload)
_, _, _, _ = c.readFrame()
})
conn := dialTest(t, server)
_, err := conn.ReadMessage()
var closeErr *CloseError
if !errors.As(err, &closeErr) || closeErr.Code != item.code {
t.Fatalf("%s: got %v", item.name, err)
}
}
}
func TestHandshakeRefusal(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "no", http.StatusForbidden)
}))
defer server.Close()
_, err := Dial(context.Background(), "ws"+strings.TrimPrefix(server.URL, "http"))
if err == nil || !strings.Contains(err.Error(), "403") {
t.Fatalf("got %v", err)
}
}
nodes/extension/README.md
+70
View File
@@ -0,0 +1,70 @@
# DevPlace Node, browser extension
A Manifest V3 extension for Chrome and Firefox from one source tree. It lets a DevPlace member donate a little of their connection to the community node network, in **fetch mode only** (an extension has no raw sockets). The protocol is `devplacepy/services/nodes/spec.py`, served at `GET /nodes/spec.json`; this extension never restates a number from it except in `src/spec.js`, which a test compares against `spec.py`.
## Install (sideload)
1. Chrome: `chrome://extensions`, enable Developer mode, Load unpacked, pick this directory. Firefox: `about:debugging#/runtime/this-firefox`, Load Temporary Add-on, pick `manifest.json`. A temporary Firefox add-on is removed on restart.
2. A disclosure page opens on first install. Read it and accept.
3. On your DevPlace profile give consent to bandwidth sharing, register a device and copy the `dpn_...` secret (shown once).
4. Open the popup, enter the server URL and the secret, grant the host permission for that server when asked, then switch sharing on.
Packaged build: `./build.sh <outdir>` writes `devplace-node-extension.zip` and `devplace-node-extension.zip.sha256`. The zip holds `manifest.json`, `src/`, `ui/` and this file; tests are not shipped.
## Chrome and Firefox from one tree
`manifest.json` declares both `background.service_worker` and `background.scripts`, both pointing at the same ES module.
| Key | Used by |
|-----|---------|
| `background.service_worker` | Chrome (service worker). Firefox ignores it with a console warning. |
| `background.scripts` | Firefox (event page). Chrome ignores it when a service worker is present. |
| `browser_specific_settings.gecko` | Firefox only (add-on id, minimum 128). Chrome warns about the unknown key and ignores it. |
Both use the `chrome.*` namespace with promises. Firefox store submission additionally needs a `data_collection_permissions` declaration, not added yet.
## How it works
- The background worker holds the WebSocket. It exchanges the device secret for a 30 second ticket (`POST /nodes/ticket`, `Authorization: Bearer dpn_...`) and opens `WS /nodes/ws?ticket=...`, then sends `hello` with `kind: "extension"` and `modes: ["fetch"]`.
- A `pong` is sent every `ping_seconds` as a keepalive and for every server `ping`. A `chrome.alarms` alarm every minute is the reconnect backstop when the worker has been suspended.
- Reconnects use full-jitter backoff between `reconnect_min_seconds` and `reconnect_max_seconds`. Close `4013` retries at the minimum delay. A kick (`4403`), a consent or device refusal (`1008` with a reason) or a rejected secret (HTTP 401/403) **halts** and shows the reason; nothing hammers the server.
- State (connection, last error) is in `chrome.storage.session`. Config (server URL, secret, limits, disclosure flag) and the usage counters are in `chrome.storage.local`.
- The secret is never logged and never shown after saving; the popup shows its first 8 characters only.
## Safety enforced in the extension, before every request
Independent of anything the server says:
- allowlist: the hosts baked into `src/spec.js` (`reddit.com`, `redd.it` and their subdomains, dot-safe suffix match). A refreshed list from `welcome` is accepted only entry by entry when it is equal or narrower (same or sub-suffix, methods and ports subsets, fetch mode present); a widening entry is dropped and reported in the popup
- https only, port 443 only (the spec's 80 is never used because plain http is refused), no credentials in the URL, method must be permitted (GET)
- `fetch` with `credentials: "omit"`, `redirect: "manual"`, `cache: "no-store"`, no referrer; only `accept` and `accept-language` are forwarded from the job, so no cookie or authorization header can be injected
- each redirect hop is re-checked against the allowlist, at most 5 hops; a redirect that leaves the allowlist is refused with `redirect_not_allowed`
- response body capped at `max_body_bytes` and by the remaining daily budget, with `truncated: true`; `set-cookie` style response headers are dropped
- timeout by `AbortController`, daily bandwidth cap, concurrency cap and active hours from the local settings, each answered with its refusal code (`timeout`, `bandwidth_exhausted`, `concurrency_exceeded`, `outside_hours`). A server value can only lower these, never raise them
Known limit: browsers hide the target of a `redirect: "manual"` response (an opaque redirect), so the extension cannot read where it points. It fails closed with `redirect_not_allowed`. A followable redirect is only handled when the response exposes its `Location`.
## Permissions
| Permission | Why |
|------------|-----|
| `storage` | config, usage counters, connection state |
| `alarms` | reconnect backstop while the worker is suspended |
| `host_permissions`: `reddit.com`, `*.reddit.com`, `redd.it`, `*.redd.it` over https | fetch the allowlisted destinations without CORS. These are exactly the destinations in `spec.py`. |
| `optional_host_permissions`: any https origin, `localhost`, `127.0.0.1` over http | requested at connect time for the one server origin you enter, so the ticket request works. Not granted at install. |
No `tabs`, `cookies`, `history`, `webRequest` or `<all_urls>` permission.
## What is and is not shared
Shared: the URL of the requested public page, the page content that comes back, your public IP address as seen by the destination, and request and byte counts. Not used: your cookies, logins, browsing history, tabs, other sites, any host outside the list, any non-https or non-443 destination. See `ui/disclosure.html`, shown on first install.
## Tests
`node --test nodes/extension/test/*.mjs` (no dependencies). They cover Policy, Limits, Fetcher with a stubbed `fetch`, WsClient with a fake WebSocket, and a drift check of `src/spec.js` against `spec.py`. Not covered by automated tests: `background.js` and the popup, which need a real browser.
## Gaps
- No icons yet; the browser shows a default.
- The `welcome` frame carries no device label, so the label in the popup is local only.
- Not tested in a real Chrome or Firefox session in this change.
nodes/extension/build.sh
Executable
+41
View File
@@ -0,0 +1,41 @@
#!/bin/sh
set -eu
if [ "$#" -ne 1 ]; then
echo "usage: build.sh <outdir>" >&2
exit 2
fi
here=$(cd "$(dirname "$0")" && pwd)
mkdir -p "$1"
out=$(cd "$1" && pwd)
name=devplace-node-extension.zip
archive="$out/$name"
rm -f "$archive"
cd "$here"
if command -v zip >/dev/null 2>&1; then
zip -q -X -r "$archive" manifest.json src ui README.md
else
python3 - "$archive" <<'PY'
import sys
import zipfile
from pathlib import Path
archive = sys.argv[1]
paths = [Path("manifest.json"), Path("README.md")]
for folder in ("src", "ui"):
paths.extend(sorted(item for item in Path(folder).rglob("*") if item.is_file()))
with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as bundle:
for path in paths:
bundle.write(path, path.as_posix())
PY
fi
cd "$out"
if command -v sha256sum >/dev/null 2>&1; then
sha256sum "$name" > "$name.sha256"
else
shasum -a 256 "$name" > "$name.sha256"
fi
cat "$name.sha256"
nodes/extension/manifest.json
+33
View File
@@ -0,0 +1,33 @@
{
"manifest_version": 3,
"name": "DevPlace Node",
"version": "0.1.0",
"description": "Donate a little of your connection to the DevPlace community node network. Fetch mode only, allowlisted hosts only, never your cookies.",
"background": {
"service_worker": "src/background.js",
"scripts": ["src/background.js"],
"type": "module"
},
"action": {
"default_title": "DevPlace Node",
"default_popup": "ui/popup.html"
},
"permissions": ["storage", "alarms"],
"host_permissions": [
"https://reddit.com/*",
"https://*.reddit.com/*",
"https://redd.it/*",
"https://*.redd.it/*"
],
"optional_host_permissions": [
"https://*/*",
"http://localhost/*",
"http://127.0.0.1/*"
],
"browser_specific_settings": {
"gecko": {
"id": "node@devplace.net",
"strict_min_version": "128.0"
}
}
}
nodes/extension/package.json
+5
View File
@@ -0,0 +1,5 @@
{
"name": "devplace-node-extension",
"private": true,
"type": "module"
}
nodes/extension/src/Config.js
+126
View File
@@ -0,0 +1,126 @@
import { LIMITS } from "./spec.js";
const KEY = "config";
const SECRET_PREFIX = "dpn_";
const SECRET_MIN_CHARS = 16;
const SECRET_MAX_CHARS = 128;
const LABEL_MAX_CHARS = 64;
const SHOWN_SECRET_CHARS = 8;
const LOOPBACK = ["localhost", "127.0.0.1", "[::1]"];
const DEFAULTS = Object.freeze({
serverUrl: "",
secret: "",
label: "",
disclosed: false,
enabled: false,
bandwidthMb: LIMITS.default_donor_mb_per_day,
maxConcurrent: LIMITS.max_concurrent_per_node,
activeFrom: "",
activeTo: "",
});
export class ConfigError extends Error {}
export class Config {
constructor(area) {
this.area = area;
this.data = { ...DEFAULTS };
}
static validateServerUrl(raw) {
let url;
try {
url = new URL(String(raw).trim());
} catch (error) {
throw new ConfigError("The server URL is not a valid URL");
}
const loopback = LOOPBACK.includes(url.hostname);
if (url.protocol !== "https:" && !(url.protocol === "http:" && loopback)) {
throw new ConfigError("The server URL must use https");
}
if (url.username || url.password) {
throw new ConfigError("The server URL must not contain credentials");
}
return url.origin;
}
static validateSecret(raw) {
const secret = String(raw).trim();
if (
!secret.startsWith(SECRET_PREFIX) ||
secret.length < SECRET_MIN_CHARS ||
secret.length > SECRET_MAX_CHARS ||
/\s/.test(secret)
) {
throw new ConfigError("That is not a DevPlace device secret (it starts with dpn_)");
}
return secret;
}
async load() {
const stored = (await this.area.get(KEY))[KEY];
this.data = { ...DEFAULTS, ...(stored && typeof stored === "object" ? stored : {}) };
}
async save() {
await this.area.set({ [KEY]: { ...this.data } });
}
get hasSecret() {
return Boolean(this.data.secret);
}
get secretPrefix() {
return this.data.secret ? this.data.secret.slice(0, SHOWN_SECRET_CHARS) + "..." : "";
}
credentials() {
return { serverUrl: this.data.serverUrl, secret: this.data.secret };
}
view() {
const { secret, ...rest } = this.data;
return { ...rest, hasSecret: this.hasSecret, secretPrefix: this.secretPrefix };
}
async connect({ serverUrl, secret, label }) {
this.data.serverUrl = Config.validateServerUrl(serverUrl);
this.data.secret = Config.validateSecret(secret);
this.data.label = String(label || "").trim().slice(0, LABEL_MAX_CHARS);
await this.save();
}
async acceptDisclosure() {
this.data.disclosed = true;
await this.save();
}
async setEnabled(enabled) {
if (enabled && !this.data.disclosed) {
throw new ConfigError("Read and accept the disclosure first");
}
if (enabled && !this.hasSecret) {
throw new ConfigError("Connect a device secret first");
}
this.data.enabled = Boolean(enabled);
await this.save();
}
async saveSettings(values) {
for (const key of ["bandwidthMb", "maxConcurrent", "activeFrom", "activeTo"]) {
if (key in values) {
this.data[key] = values[key];
}
}
await this.save();
}
async forget() {
this.data.secret = "";
this.data.serverUrl = "";
this.data.label = "";
this.data.enabled = false;
await this.save();
}
}
nodes/extension/src/Fetcher.js
+193
View File
@@ -0,0 +1,193 @@
import {
DROPPED_RESPONSE_HEADERS,
FETCH_FAILED,
FORWARDED_REQUEST_HEADERS,
MAX_BODY_CHUNKS,
MAX_HEADER_VALUE_CHARS,
MAX_REDIRECTS,
MAX_RESPONSE_HEADERS,
REFUSALS,
} from "./spec.js";
const REDIRECT_STATUSES = [301, 302, 303, 307, 308];
const REQUEST_OVERHEAD_BYTES = 256;
const BASE64_CHUNK = 0x8000;
export function toBase64(bytes) {
let text = "";
for (let index = 0; index < bytes.length; index += BASE64_CHUNK) {
text += String.fromCharCode.apply(null, bytes.subarray(index, index + BASE64_CHUNK));
}
return btoa(text);
}
function concat(parts, total) {
const joined = new Uint8Array(total);
let offset = 0;
for (const part of parts) {
joined.set(part, offset);
offset += part.length;
}
return joined;
}
export class Fetcher {
constructor({ policy, limits, usage, fetchImpl, clock }) {
this.policy = policy;
this.limits = limits;
this.usage = usage;
this.fetchImpl = fetchImpl || ((...args) => globalThis.fetch(...args));
this.clock = clock || (() => Date.now());
}
setPolicy(policy) {
this.policy = policy;
}
failure(jobId, code) {
return { type: "job_error", job_id: jobId, code };
}
requestHeaders(raw) {
const headers = {};
if (!raw || typeof raw !== "object") {
return headers;
}
for (const [name, value] of Object.entries(raw)) {
const key = name.toLowerCase();
if (FORWARDED_REQUEST_HEADERS.includes(key) && typeof value === "string") {
headers[key] = value;
}
}
return headers;
}
responseHeaders(response) {
const headers = {};
let count = 0;
response.headers.forEach((value, name) => {
const key = name.toLowerCase();
if (DROPPED_RESPONSE_HEADERS.includes(key) || count >= MAX_RESPONSE_HEADERS) {
return;
}
headers[key] = String(value).slice(0, MAX_HEADER_VALUE_CHARS);
count += 1;
});
return headers;
}
isRedirect(response) {
return response.type === "opaqueredirect" || REDIRECT_STATUSES.includes(response.status);
}
async readBody(response, cap) {
if (!response.body || typeof response.body.getReader !== "function") {
const whole = new Uint8Array(await response.arrayBuffer());
return whole.length > cap
? { bytes: whole.slice(0, cap), truncated: true }
: { bytes: whole, truncated: false };
}
const reader = response.body.getReader();
const parts = [];
let total = 0;
let truncated = false;
for (let chunks = 0; chunks < MAX_BODY_CHUNKS; chunks += 1) {
const { done, value } = await reader.read();
if (done) {
return { bytes: concat(parts, total), truncated };
}
const room = cap - total;
if (value.length > room) {
parts.push(value.subarray(0, room));
total += room;
truncated = true;
break;
}
parts.push(value);
total += value.length;
}
truncated = true;
await reader.cancel();
return { bytes: concat(parts, total), truncated };
}
async follow(job, verdict, signal) {
let current = verdict.url;
const headers = this.requestHeaders(job.headers);
for (let hop = 0; hop <= MAX_REDIRECTS; hop += 1) {
const response = await this.fetchImpl(current.href, {
method: verdict.method,
headers,
credentials: "omit",
redirect: "manual",
cache: "no-store",
referrerPolicy: "no-referrer",
signal,
});
if (!this.isRedirect(response)) {
return { response, final: current };
}
const location = response.headers.get("location");
if (response.type === "opaqueredirect" || !location || hop === MAX_REDIRECTS) {
if (response.body && typeof response.body.cancel === "function") {
await response.body.cancel();
}
return { refused: REFUSALS.REDIRECT_NOT_ALLOWED };
}
const next = this.policy.checkRedirect(location, current, verdict.method);
if (response.body && typeof response.body.cancel === "function") {
await response.body.cancel();
}
if (!next.ok) {
return { refused: REFUSALS.REDIRECT_NOT_ALLOWED };
}
current = next.url;
}
return { refused: REFUSALS.REDIRECT_NOT_ALLOWED };
}
async execute(job) {
const verdict = this.policy.check(job.method, job.url);
if (!verdict.ok) {
return this.failure(job.job_id, verdict.code);
}
const refusal = this.limits.admit(this.usage.usedBytes);
if (refusal) {
return this.failure(job.job_id, refusal);
}
const started = this.clock();
const up = job.url.length + REQUEST_OVERHEAD_BYTES;
const controller = new AbortController();
const timer = setTimeout(
() => controller.abort(),
this.limits.timeoutSeconds(job.timeout) * 1000,
);
try {
const outcome = await this.follow(job, verdict, controller.signal);
if (outcome.refused) {
await this.usage.record({ ok: false, up, down: 0 });
return this.failure(job.job_id, outcome.refused);
}
const budget = this.limits.remainingBytes(this.usage.usedBytes + up);
const cap = Math.min(this.limits.maxBodyBytes, budget);
const body = await this.readBody(outcome.response, cap);
const down = body.bytes.length;
await this.usage.record({ ok: true, up, down });
return {
type: "job_result",
job_id: job.job_id,
status: outcome.response.status,
headers: this.responseHeaders(outcome.response),
body_b64: toBase64(body.bytes),
truncated: body.truncated,
elapsed_ms: Math.max(0, Math.round(this.clock() - started)),
};
} catch (error) {
await this.usage.record({ ok: false, up, down: 0 });
return this.failure(job.job_id, controller.signal.aborted ? REFUSALS.TIMEOUT : FETCH_FAILED);
} finally {
clearTimeout(timer);
this.limits.release();
}
}
}
nodes/extension/src/Limits.js
+167
View File
@@ -0,0 +1,167 @@
import { LIMITS, REFUSALS } from "./spec.js";
const MB = 1024 * 1024;
const HOURS = /^([01]\d|2[0-3]):([0-5]\d)$/;
const CAP_KEYS = [
"max_frame_bytes",
"max_body_bytes",
"max_concurrent_per_node",
"job_timeout_seconds",
"ping_seconds",
"url_max_chars",
];
function isCount(value) {
return typeof value === "number" && Number.isInteger(value) && value >= 1;
}
function clampInt(value, low, high, fallback) {
if (!isCount(value)) {
return fallback;
}
return Math.min(high, Math.max(low, value));
}
function minutes(text) {
const match = HOURS.exec(text);
return match ? Number(match[1]) * 60 + Number(match[2]) : null;
}
export class Limits {
constructor(options = {}) {
this.baked = options.baked || LIMITS;
this.clock = options.clock || (() => new Date());
this.caps = {};
for (const key of CAP_KEYS) {
this.caps[key] = this.baked[key];
}
this.reconnectMin = this.baked.reconnect_min_seconds;
this.reconnectMax = this.baked.reconnect_max_seconds;
this.serverBandwidthMb = this.baked.max_donor_mb_per_day;
this.serverConcurrent = this.baked.max_concurrent_per_node;
this.local = Limits.normaliseLocal({}, this.baked);
this.inflight = 0;
if (options.local) {
this.setLocal(options.local);
}
}
static normaliseLocal(values, baked = LIMITS) {
const from = minutes(values.activeFrom || "");
const to = minutes(values.activeTo || "");
const both = from !== null && to !== null;
return {
bandwidthMb: clampInt(
values.bandwidthMb,
1,
baked.max_donor_mb_per_day,
baked.default_donor_mb_per_day,
),
maxConcurrent: clampInt(
values.maxConcurrent,
1,
baked.max_concurrent_per_node,
baked.max_concurrent_per_node,
),
activeFrom: both ? values.activeFrom : "",
activeTo: both ? values.activeTo : "",
};
}
setLocal(values) {
this.local = Limits.normaliseLocal({ ...this.local, ...values }, this.baked);
}
applyWelcome(welcomeLimits, serverSettings) {
const incoming = welcomeLimits && typeof welcomeLimits === "object" ? welcomeLimits : {};
for (const key of CAP_KEYS) {
this.caps[key] = Math.min(this.baked[key], isCount(incoming[key]) ? incoming[key] : this.baked[key]);
}
const ceiling = this.baked.reconnect_max_seconds;
this.reconnectMin = clampInt(incoming.reconnect_min_seconds, 1, ceiling, this.baked.reconnect_min_seconds);
this.reconnectMax = clampInt(incoming.reconnect_max_seconds, this.reconnectMin, ceiling, ceiling);
this.applyServerSettings(serverSettings);
}
applyServerSettings(settings) {
const values = settings && typeof settings === "object" ? settings : {};
this.serverBandwidthMb = clampInt(
values.bandwidth_mb_day,
1,
this.baked.max_donor_mb_per_day,
this.baked.max_donor_mb_per_day,
);
this.serverConcurrent = clampInt(
values.max_concurrent,
1,
this.baked.max_concurrent_per_node,
this.baked.max_concurrent_per_node,
);
}
get maxBodyBytes() {
return this.caps.max_body_bytes;
}
get maxFrameBytes() {
return this.caps.max_frame_bytes;
}
get urlMaxChars() {
return this.caps.url_max_chars;
}
get pingSeconds() {
return this.caps.ping_seconds;
}
get concurrency() {
return Math.min(this.caps.max_concurrent_per_node, this.local.maxConcurrent, this.serverConcurrent);
}
get bandwidthBytes() {
return Math.min(this.local.bandwidthMb, this.serverBandwidthMb) * MB;
}
timeoutSeconds(requested) {
const asked = isCount(requested) ? requested : this.caps.job_timeout_seconds;
return Math.min(asked, this.caps.job_timeout_seconds);
}
helloLimits() {
return { max_concurrent: this.concurrency };
}
remainingBytes(usedBytes) {
return Math.max(0, this.bandwidthBytes - usedBytes);
}
inActiveHours() {
const from = minutes(this.local.activeFrom);
const to = minutes(this.local.activeTo);
if (from === null || to === null || from === to) {
return true;
}
const now = this.clock();
const current = now.getHours() * 60 + now.getMinutes();
return from < to ? current >= from && current < to : current >= from || current < to;
}
admit(usedBytes) {
if (!this.inActiveHours()) {
return REFUSALS.OUTSIDE_HOURS;
}
if (this.remainingBytes(usedBytes) <= 0) {
return REFUSALS.BANDWIDTH_EXHAUSTED;
}
if (this.inflight >= this.concurrency) {
return REFUSALS.CONCURRENCY_EXCEEDED;
}
this.inflight += 1;
return null;
}
release() {
this.inflight = Math.max(0, this.inflight - 1);
}
}
nodes/extension/src/Policy.js
+128
View File
@@ -0,0 +1,128 @@
import {
ALLOWED_PORTS,
DESTINATIONS,
HTTPS_PORT,
LIMITS,
MODE_FETCH,
REFUSALS,
} from "./spec.js";
const HTTPS = "https:";
function isSubset(candidate, allowed) {
return Array.isArray(candidate) && candidate.every((item) => allowed.includes(item));
}
function normaliseHost(host) {
return String(host).trim().toLowerCase().replace(/\.+$/, "");
}
export class Policy {
constructor(destinations = DESTINATIONS, limits = LIMITS) {
this.destinations = destinations;
this.urlMaxChars = limits.url_max_chars;
}
static narrow(baked, refreshed) {
const accepted = [];
const rejected = [];
if (!Array.isArray(refreshed)) {
return { destinations: [], rejected: ["destinations is not a list"] };
}
for (const item of refreshed) {
const suffix = item && typeof item.suffix === "string" ? normaliseHost(item.suffix) : "";
const parent = baked.find(
(entry) => suffix === entry.suffix || suffix.endsWith("." + entry.suffix),
);
if (!parent) {
rejected.push(`${suffix || "?"}: not in the baked-in allowlist`);
continue;
}
const methods = Array.isArray(item.methods) ? item.methods.map((m) => String(m).toUpperCase()) : [];
const ports = Array.isArray(item.ports) ? item.ports : [];
const modes = Array.isArray(item.modes) ? item.modes : [];
if (!methods.length || !isSubset(methods, parent.methods)) {
rejected.push(`${suffix}: methods widen the baked-in list`);
continue;
}
if (!ports.length || !isSubset(ports, parent.ports) || !isSubset(ports, ALLOWED_PORTS)) {
rejected.push(`${suffix}: ports widen the baked-in list`);
continue;
}
if (!modes.includes(MODE_FETCH)) {
rejected.push(`${suffix}: not available in fetch mode`);
continue;
}
accepted.push({
suffix,
adapter: String(item.adapter || parent.adapter),
methods,
ports,
modes,
max_rps: typeof item.max_rps === "number" ? item.max_rps : parent.max_rps,
});
}
return { destinations: accepted, rejected };
}
static fromRefreshed(baked, refreshed, limits = LIMITS) {
const narrowed = Policy.narrow(baked, refreshed);
return { policy: new Policy(narrowed.destinations, limits), rejected: narrowed.rejected };
}
destinationFor(host) {
const name = normaliseHost(host);
return (
this.destinations.find((item) => name === item.suffix || name.endsWith("." + item.suffix)) || null
);
}
refuse(code, reason) {
return { ok: false, code, reason };
}
check(method, rawUrl) {
if (typeof rawUrl !== "string" || !rawUrl || rawUrl.length > this.urlMaxChars) {
return this.refuse(REFUSALS.HOST_NOT_ALLOWED, "url is missing or too long");
}
let url;
try {
url = new URL(rawUrl);
} catch (error) {
return this.refuse(REFUSALS.HOST_NOT_ALLOWED, "url does not parse");
}
if (url.protocol !== HTTPS) {
return this.refuse(REFUSALS.HOST_NOT_ALLOWED, "only https is allowed");
}
if (url.username || url.password) {
return this.refuse(REFUSALS.HOST_NOT_ALLOWED, "credentials in the url are refused");
}
const destination = this.destinationFor(url.hostname);
if (!destination) {
return this.refuse(REFUSALS.HOST_NOT_ALLOWED, `${url.hostname} is not on the allowlist`);
}
const port = url.port === "" ? HTTPS_PORT : Number(url.port);
if (port !== HTTPS_PORT || !destination.ports.includes(port) || !ALLOWED_PORTS.includes(port)) {
return this.refuse(REFUSALS.PORT_NOT_ALLOWED, `port ${port} is not allowed over https`);
}
const verb = String(method || "").toUpperCase();
if (!destination.methods.includes(verb)) {
return this.refuse(REFUSALS.METHOD_NOT_ALLOWED, `${verb || "?"} is not permitted`);
}
return { ok: true, url, destination, method: verb };
}
checkRedirect(location, base, method) {
let target;
try {
target = new URL(location, base);
} catch (error) {
return this.refuse(REFUSALS.REDIRECT_NOT_ALLOWED, "redirect target does not parse");
}
const verdict = this.check(method, target.href);
if (!verdict.ok) {
return this.refuse(REFUSALS.REDIRECT_NOT_ALLOWED, verdict.reason);
}
return verdict;
}
}
nodes/extension/src/State.js
+37
View File
@@ -0,0 +1,37 @@
const KEY = "state";
export const STATES = Object.freeze({
OFF: "off",
CONNECTING: "connecting",
ONLINE: "online",
BACKOFF: "backoff",
HALTED: "halted",
});
export class State {
constructor(area, clock = () => Date.now()) {
this.area = area;
this.clock = clock;
this.data = { state: STATES.OFF, detail: "", since: clock(), deviceUid: "", lastError: "" };
this.chain = Promise.resolve();
}
async load() {
const stored = (await this.area.get(KEY))[KEY];
if (stored && typeof stored === "object") {
this.data = { ...this.data, ...stored };
}
}
snapshot() {
return { ...this.data };
}
set(patch) {
const changed = patch.state && patch.state !== this.data.state;
this.data = { ...this.data, ...patch, since: changed ? this.clock() : this.data.since };
const payload = { ...this.data };
this.chain = this.chain.then(() => this.area.set({ [KEY]: payload }));
return this.chain;
}
}
nodes/extension/src/Usage.js
+81
View File
@@ -0,0 +1,81 @@
const KEY = "usage";
function dayKey(date) {
const month = String(date.getMonth() + 1).padStart(2, "0");
const day = String(date.getDate()).padStart(2, "0");
return `${date.getFullYear()}-${month}-${day}`;
}
function empty(day) {
return {
day,
requests: 0,
failed: 0,
bytesUp: 0,
bytesDown: 0,
lifetimeRequests: 0,
lifetimeBytes: 0,
};
}
export class Usage {
constructor(area, clock = () => new Date()) {
this.area = area;
this.clock = clock;
this.data = empty(dayKey(clock()));
this.chain = Promise.resolve();
}
async load() {
const stored = (await this.area.get(KEY))[KEY];
if (stored && typeof stored === "object") {
this.data = { ...empty(dayKey(this.clock())), ...stored };
}
this.roll();
}
roll() {
const today = dayKey(this.clock());
if (this.data.day !== today) {
this.data = {
...empty(today),
lifetimeRequests: this.data.lifetimeRequests,
lifetimeBytes: this.data.lifetimeBytes,
};
}
}
snapshot() {
this.roll();
return { ...this.data };
}
get usedBytes() {
this.roll();
return this.data.bytesUp + this.data.bytesDown;
}
record({ ok, up, down }) {
this.roll();
const bytesUp = Math.max(0, Math.floor(up) || 0);
const bytesDown = Math.max(0, Math.floor(down) || 0);
this.data.requests += 1;
this.data.failed += ok ? 0 : 1;
this.data.bytesUp += bytesUp;
this.data.bytesDown += bytesDown;
this.data.lifetimeRequests += 1;
this.data.lifetimeBytes += bytesUp + bytesDown;
return this.persist();
}
persist() {
const payload = { ...this.data };
this.chain = this.chain.then(() => this.area.set({ [KEY]: payload }));
return this.chain;
}
async reset() {
this.data = empty(dayKey(this.clock()));
await this.persist();
}
}
nodes/extension/src/WsClient.js
+375
View File
@@ -0,0 +1,375 @@
import { Policy } from "./Policy.js";
import { STATES } from "./State.js";
import {
CLIENT_KIND,
CLOSE_CAPACITY,
CLOSE_KICKED,
CLOSE_NORMAL,
CLOSE_PROTOCOL,
CLOSE_RETRY_OWNER,
DESTINATIONS,
FRAME_CEILING_BYTES,
HALTING_CLOSE_REASONS_EXEMPT,
MAX_BACKOFF_EXPONENT,
MODE_FETCH,
PROTOCOL_VERSION,
REFUSALS,
SERVER_FRAMES,
} from "./spec.js";
const OPEN = 1;
const POLICY_CLOSE = 1008;
const TICKET_PATH = "/nodes/ticket";
const WS_PATH = "/nodes/ws";
const HALT_STATUSES = [401, 403];
class FrameFault extends Error {}
export class WsClient {
constructor(options) {
this.config = options.config;
this.limits = options.limits;
this.fetcher = options.fetcher;
this.state = options.state;
this.fetchImpl = options.fetchImpl || ((...args) => globalThis.fetch(...args));
this.WebSocketImpl = options.WebSocketImpl || globalThis.WebSocket;
this.timers = options.timers || globalThis;
this.random = options.random || Math.random;
this.clientVersion = options.clientVersion || "0.0.0";
this.platform = options.platform || "chrome";
this.baked = options.baked || DESTINATIONS;
this.socket = null;
this.wanted = false;
this.connecting = false;
this.welcomed = false;
this.attempt = 0;
this.retryTimer = null;
this.pingTimer = null;
this.pending = new Set();
this.generation = 0;
}
async start() {
this.wanted = true;
await this.ensure();
}
async stop(detail = "") {
this.wanted = false;
this.clearTimers();
this.generation += 1;
this.closeSocket(CLOSE_NORMAL, "stop");
await this.state.set({ state: STATES.OFF, detail });
}
async ensure() {
if (!this.wanted || this.socket || this.connecting || this.retryTimer) {
return;
}
await this.connectOnce();
}
clearTimers() {
if (this.retryTimer) {
this.timers.clearTimeout(this.retryTimer);
this.retryTimer = null;
}
if (this.pingTimer) {
this.timers.clearInterval(this.pingTimer);
this.pingTimer = null;
}
}
closeSocket(code, reason) {
const socket = this.socket;
this.socket = null;
if (socket && socket.readyState <= OPEN) {
socket.close(code, reason);
}
}
backoffSeconds(fast) {
const low = this.limits.reconnectMin;
if (fast) {
return low;
}
const exponent = Math.min(this.attempt, MAX_BACKOFF_EXPONENT);
const ceiling = Math.min(this.limits.reconnectMax, low * 2 ** exponent);
return Math.max(low, this.random() * ceiling);
}
async scheduleRetry(detail, fast = false) {
if (!this.wanted) {
return;
}
const seconds = this.backoffSeconds(fast);
this.attempt += 1;
this.clearTimers();
this.retryTimer = this.timers.setTimeout(() => {
this.retryTimer = null;
this.ensure();
}, seconds * 1000);
await this.state.set({ state: STATES.BACKOFF, detail, retryInSeconds: Math.round(seconds) });
}
async halt(detail) {
this.wanted = false;
this.clearTimers();
this.generation += 1;
this.closeSocket(CLOSE_NORMAL, "halt");
await this.state.set({ state: STATES.HALTED, detail, lastError: detail });
}
async fetchTicket() {
const { serverUrl, secret } = this.config.credentials();
const response = await this.fetchImpl(serverUrl + TICKET_PATH, {
method: "POST",
headers: { Authorization: `Bearer ${secret}` },
credentials: "omit",
cache: "no-store",
});
if (!response.ok) {
let reason = "";
try {
reason = (await response.json()).reason || "";
} catch (error) {
reason = "";
}
const error = new Error(`ticket refused with ${response.status} ${reason}`.trim());
error.status = response.status;
throw error;
}
const body = await response.json();
if (!body || typeof body.ticket !== "string" || !body.ticket) {
throw new Error("ticket response has no ticket");
}
const target = new URL(typeof body.ws_path === "string" ? body.ws_path : WS_PATH, serverUrl);
if (target.origin !== new URL(serverUrl).origin) {
throw new Error("ticket response points at another origin");
}
target.protocol = target.protocol === "https:" ? "wss:" : "ws:";
target.searchParams.set("ticket", body.ticket);
return target.href;
}
async connectOnce() {
this.connecting = true;
const generation = this.generation;
await this.state.set({ state: STATES.CONNECTING, detail: "" });
let address;
try {
address = await this.fetchTicket();
} catch (error) {
this.connecting = false;
if (generation !== this.generation) {
return;
}
if (HALT_STATUSES.includes(error.status)) {
await this.halt(error.message);
return;
}
await this.state.set({ lastError: error.message });
await this.scheduleRetry(error.message);
return;
}
this.connecting = false;
if (generation !== this.generation) {
return;
}
this.open(address, generation);
}
open(address, generation) {
this.welcomed = false;
const socket = new this.WebSocketImpl(address);
this.socket = socket;
socket.onopen = () => {
if (generation !== this.generation || this.socket !== socket) {
return;
}
this.sendFrame("hello", {
protocol: PROTOCOL_VERSION,
client_version: this.clientVersion,
kind: CLIENT_KIND,
platform: this.platform,
modes: [MODE_FETCH],
limits: this.limits.helloLimits(),
});
};
socket.onmessage = (event) => {
if (this.socket !== socket) {
return;
}
return this.handleMessage(event.data).catch((error) => this.protocolFault(error));
};
socket.onclose = (event) => {
if (this.socket !== socket) {
return;
}
this.socket = null;
this.onClosed(event.code, event.reason || "");
};
socket.onerror = () => {
this.state.set({ lastError: "websocket error" });
};
}
sendFrame(type, fields = {}) {
if (!this.socket || this.socket.readyState !== OPEN) {
throw new Error(`cannot send ${type}, socket is not open`);
}
const text = JSON.stringify({ type, ...fields });
const ceiling = FRAME_CEILING_BYTES[type] || FRAME_CEILING_BYTES.default;
if (new TextEncoder().encode(text).length > ceiling) {
throw new Error(`${type} frame exceeds its size cap`);
}
this.socket.send(text);
}
async protocolFault(error) {
await this.state.set({ lastError: error.message });
this.closeSocket(CLOSE_PROTOCOL, "protocol");
this.clearTimers();
await this.scheduleRetry(error.message);
}
decode(raw) {
if (typeof raw !== "string") {
throw new FrameFault("binary frames are not part of the protocol");
}
if (raw.length > this.limits.maxFrameBytes) {
throw new FrameFault("server frame exceeds the size cap");
}
let frame;
try {
frame = JSON.parse(raw);
} catch (error) {
throw new FrameFault("server frame is not valid JSON");
}
if (!frame || typeof frame !== "object" || Array.isArray(frame)) {
throw new FrameFault("server frame must be a JSON object");
}
const shape = SERVER_FRAMES[frame.type];
if (!shape) {
throw new FrameFault(`unknown server frame ${String(frame.type)}`);
}
for (const [name, kind] of Object.entries(shape)) {
const value = frame[name];
const good = kind === "array" ? Array.isArray(value) : typeof value === kind && value !== null;
if (!good) {
throw new FrameFault(`frame ${frame.type} field ${name} must be ${kind}`);
}
}
return frame;
}
async handleMessage(raw) {
const frame = this.decode(raw);
if (frame.type !== "welcome" && !this.welcomed && frame.type !== "kick") {
throw new FrameFault(`${frame.type} arrived before welcome`);
}
if (frame.type === "welcome") {
await this.onWelcome(frame);
} else if (frame.type === "ping") {
this.sendFrame("pong");
} else if (frame.type === "job") {
this.runJob(frame);
} else if (frame.type === "settings_update") {
this.limits.applyServerSettings(frame.settings);
} else if (frame.type === "kick") {
await this.halt(`kicked by the server: ${frame.reason}`);
} else {
this.sendFrame("tunnel_opened", {
conn_id: frame.conn_id,
ok: false,
code: REFUSALS.HOST_NOT_ALLOWED,
});
throw new FrameFault(`${frame.type} is a tunnel frame and this client is fetch-only`);
}
}
async onWelcome(frame) {
if (this.welcomed) {
throw new FrameFault("a second welcome arrived");
}
this.limits.applyWelcome(frame.limits, frame.settings);
const refreshed = Policy.fromRefreshed(this.baked, frame.destinations, this.limits.baked);
this.fetcher.setPolicy(refreshed.policy);
this.welcomed = true;
this.attempt = 0;
this.startKeepalive();
const notes = refreshed.rejected.length ? `ignored: ${refreshed.rejected.join("; ")}` : "";
await this.state.set({
state: STATES.ONLINE,
detail: notes,
deviceUid: frame.device_uid,
lastError: notes,
});
}
startKeepalive() {
if (this.pingTimer) {
this.timers.clearInterval(this.pingTimer);
}
this.pingTimer = this.timers.setInterval(() => {
try {
this.sendFrame("pong");
} catch (error) {
this.state.set({ lastError: error.message });
}
}, this.limits.pingSeconds * 1000);
}
runJob(frame) {
const task = this.fetcher
.execute(frame)
.then((result) => {
this.sendFrame(result.type, result);
return this.reportUsage(result);
})
.catch((error) => this.state.set({ lastError: error.message }))
.finally(() => this.pending.delete(task));
this.pending.add(task);
}
reportUsage(result) {
const down = result.type === "job_result" ? Math.ceil((result.body_b64.length * 3) / 4) : 0;
this.sendFrame("usage", { bytes_up: 0, bytes_down: down });
}
async idle() {
while (this.pending.size) {
await Promise.all([...this.pending]);
}
}
classifyClose(code, reason) {
if (code === CLOSE_KICKED) {
return { action: "halt", detail: `closed by the server: ${reason || "kicked"}` };
}
if (code === CLOSE_RETRY_OWNER) {
return { action: "retry", fast: true, detail: "server asked for a retry" };
}
if (code === POLICY_CLOSE && !HALTING_CLOSE_REASONS_EXEMPT.includes(reason)) {
return { action: "halt", detail: `refused by the server: ${reason || "policy"}` };
}
if (code === CLOSE_CAPACITY) {
return { action: "retry", fast: false, detail: "server at capacity" };
}
return { action: "retry", fast: false, detail: `closed ${code} ${reason}`.trim() };
}
async onClosed(code, reason) {
this.clearTimers();
this.welcomed = false;
if (!this.wanted) {
return;
}
const verdict = this.classifyClose(code, reason);
if (verdict.action === "halt") {
await this.halt(verdict.detail);
return;
}
await this.scheduleRetry(verdict.detail, verdict.fast);
}
}
nodes/extension/src/background.js
+135
View File
@@ -0,0 +1,135 @@
import { Config, ConfigError } from "./Config.js";
import { Fetcher } from "./Fetcher.js";
import { Limits } from "./Limits.js";
import { Policy } from "./Policy.js";
import { State, STATES } from "./State.js";
import { Usage } from "./Usage.js";
import { WsClient } from "./WsClient.js";
const ALARM = "devplace-node-backstop";
const ALARM_PERIOD_MINUTES = 1;
const DISCLOSURE_PAGE = "ui/disclosure.html";
const config = new Config(chrome.storage.local);
const usage = new Usage(chrome.storage.local);
const state = new State(chrome.storage.session);
const limits = new Limits();
const fetcher = new Fetcher({ policy: new Policy(), limits, usage });
const client = new WsClient({
config,
limits,
fetcher,
state,
clientVersion: chrome.runtime.getManifest().version,
platform: navigator.userAgent.includes("Firefox") ? "firefox" : "chrome",
});
let ready = null;
function boot() {
if (!ready) {
ready = (async () => {
await Promise.all([config.load(), usage.load(), state.load()]);
limits.setLocal(config.data);
})();
}
return ready;
}
async function reconcile() {
await boot();
limits.setLocal(config.data);
if (config.data.enabled && config.data.disclosed && config.hasSecret) {
await client.start();
} else if (client.wanted || state.data.state !== STATES.OFF) {
await client.stop();
}
}
async function status() {
await boot();
return {
config: config.view(),
state: state.snapshot(),
usage: usage.snapshot(),
live: { concurrency: limits.concurrency, bandwidthBytes: limits.bandwidthBytes },
};
}
const COMMANDS = {
status: async () => status(),
connect: async (message) => {
await config.connect(message);
await client.stop("credentials changed");
await reconcile();
return status();
},
accept: async () => {
await config.acceptDisclosure();
return status();
},
enable: async (message) => {
await config.setEnabled(Boolean(message.enabled));
await reconcile();
return status();
},
settings: async (message) => {
await config.saveSettings(message.values || {});
limits.setLocal(config.data);
return status();
},
forget: async () => {
await config.forget();
await client.stop("device forgotten");
return status();
},
resetUsage: async () => {
await usage.reset();
return status();
},
};
async function dispatch(message) {
await boot();
const command = COMMANDS[message && message.cmd];
if (!command) {
return { error: `unknown command ${String(message && message.cmd)}` };
}
try {
return await command(message);
} catch (error) {
if (error instanceof ConfigError) {
return { error: error.message };
}
await state.set({ lastError: error.message });
return { error: error.message };
}
}
chrome.runtime.onMessage.addListener((message, sender, respond) => {
if (sender.id !== chrome.runtime.id) {
return false;
}
dispatch(message).then(respond);
return true;
});
chrome.alarms.onAlarm.addListener((alarm) => {
if (alarm.name === ALARM) {
reconcile().catch((error) => state.set({ lastError: error.message }));
}
});
chrome.runtime.onInstalled.addListener((details) => {
chrome.alarms.create(ALARM, { periodInMinutes: ALARM_PERIOD_MINUTES });
if (details.reason === "install") {
chrome.tabs.create({ url: chrome.runtime.getURL(DISCLOSURE_PAGE) });
}
});
chrome.runtime.onStartup.addListener(() => {
chrome.alarms.create(ALARM, { periodInMinutes: ALARM_PERIOD_MINUTES });
reconcile().catch((error) => state.set({ lastError: error.message }));
});
reconcile().catch((error) => state.set({ lastError: error.message }));
nodes/extension/src/spec.js
+104
View File
@@ -0,0 +1,104 @@
export const PROTOCOL_VERSION = 1;
export const MIN_SUPPORTED_PROTOCOL = 1;
export const CLIENT_KIND = "extension";
export const MODE_FETCH = "fetch";
export const ALLOWED_PORTS = Object.freeze([80, 443]);
export const HTTPS_PORT = 443;
export const CLOSE_NORMAL = 1000;
export const CLOSE_PROTOCOL = 4400;
export const CLOSE_RETRY_OWNER = 4013;
export const CLOSE_CAPACITY = 4029;
export const CLOSE_KICKED = 4403;
export const CLOSE_POLICY = 1008;
export const MAX_REDIRECTS = 5;
export const MAX_RESPONSE_HEADERS = 64;
export const MAX_HEADER_VALUE_CHARS = 512;
export const MAX_BODY_CHUNKS = 65536;
export const MAX_BACKOFF_EXPONENT = 16;
export const LIMITS = Object.freeze({
max_frame_bytes: 256 * 1024,
max_body_bytes: 2 * 1024 * 1024,
max_concurrent_per_node: 2,
job_timeout_seconds: 25,
default_donor_mb_per_day: 500,
max_donor_mb_per_day: 20 * 1024,
ping_seconds: 20,
reconnect_min_seconds: 1,
reconnect_max_seconds: 60,
url_max_chars: 2048,
});
export const FRAME_CEILING_BYTES = Object.freeze({
job_result: 3 * 1024 * 1024,
default: 256 * 1024,
});
export const DESTINATIONS = Object.freeze([
Object.freeze({
suffix: "reddit.com",
adapter: "reddit",
methods: Object.freeze(["GET"]),
ports: Object.freeze([80, 443]),
modes: Object.freeze(["tunnel", "fetch"]),
max_rps: 0.5,
}),
Object.freeze({
suffix: "redd.it",
adapter: "reddit",
methods: Object.freeze(["GET"]),
ports: Object.freeze([80, 443]),
modes: Object.freeze(["tunnel", "fetch"]),
max_rps: 0.5,
}),
]);
export const REFUSALS = Object.freeze({
HOST_NOT_ALLOWED: "host_not_allowed",
PORT_NOT_ALLOWED: "port_not_allowed",
ADDRESS_NOT_PUBLIC: "address_not_public",
REDIRECT_NOT_ALLOWED: "redirect_not_allowed",
METHOD_NOT_ALLOWED: "method_not_allowed",
BANDWIDTH_EXHAUSTED: "bandwidth_exhausted",
CONCURRENCY_EXCEEDED: "concurrency_exceeded",
OUTSIDE_HOURS: "outside_hours",
BODY_TOO_LARGE: "body_too_large",
TIMEOUT: "timeout",
});
export const FETCH_FAILED = "fetch_failed";
export const SERVER_FRAMES = Object.freeze({
welcome: Object.freeze({
device_uid: "string",
limits: "object",
destinations: "array",
settings: "object",
}),
ping: Object.freeze({}),
job: Object.freeze({
job_id: "string",
method: "string",
url: "string",
timeout: "number",
}),
tunnel_open: Object.freeze({ conn_id: "string", host: "string", port: "number" }),
tunnel_data: Object.freeze({ conn_id: "string", b64: "string" }),
tunnel_close: Object.freeze({ conn_id: "string" }),
settings_update: Object.freeze({ settings: "object" }),
kick: Object.freeze({ reason: "string" }),
});
export const DROPPED_RESPONSE_HEADERS = Object.freeze([
"set-cookie",
"set-cookie2",
"authorization",
"proxy-authenticate",
]);
export const FORWARDED_REQUEST_HEADERS = Object.freeze(["accept", "accept-language"]);
export const HALTING_CLOSE_REASONS_EXEMPT = Object.freeze(["ticket", "heartbeat", "error", "capacity"]);
nodes/extension/test/fetcher.test.mjs
+197
View File
@@ -0,0 +1,197 @@
import test from "node:test";
import assert from "node:assert/strict";
import { Fetcher } from "../src/Fetcher.js";
import { Limits } from "../src/Limits.js";
import { Policy } from "../src/Policy.js";
import { Usage } from "../src/Usage.js";
import { MAX_REDIRECTS, REFUSALS } from "../src/spec.js";
function memoryArea() {
const store = {};
return {
async get(key) {
return key in store ? { [key]: store[key] } : {};
},
async set(values) {
Object.assign(store, values);
},
};
}
function build(fetchImpl, local = {}) {
const limits = new Limits({ local });
const usage = new Usage(memoryArea());
const fetcher = new Fetcher({ policy: new Policy(), limits, usage, fetchImpl });
return { fetcher, limits, usage };
}
const job = (url = "https://www.reddit.com/r/x.json", extra = {}) => ({
job_id: "j1",
method: "GET",
url,
timeout: 5,
...extra,
});
function response(body, init = {}) {
return new Response(body, { status: 200, ...init });
}
test("success returns a job_result with base64 body and no donor credentials", async () => {
const calls = [];
const { fetcher, usage } = build(async (url, options) => {
calls.push({ url, options });
return response("hello", { headers: { "content-type": "text/plain", "set-cookie": "a=b" } });
});
const result = await fetcher.execute(
job(undefined, { headers: { Cookie: "x=1", Authorization: "Bearer z", Accept: "text/html" } }),
);
assert.equal(result.type, "job_result");
assert.equal(result.status, 200);
assert.equal(Buffer.from(result.body_b64, "base64").toString(), "hello");
assert.equal(result.truncated, false);
assert.equal(result.headers["content-type"], "text/plain");
assert.equal("set-cookie" in result.headers, false);
assert.equal(calls[0].options.credentials, "omit");
assert.equal(calls[0].options.redirect, "manual");
assert.deepEqual(calls[0].options.headers, { accept: "text/html" });
assert.equal(usage.snapshot().requests, 1);
assert.equal(usage.snapshot().bytesDown, 5);
});
test("a disallowed url never reaches fetch", async () => {
let called = 0;
const { fetcher, limits } = build(async () => {
called += 1;
return response("x");
});
for (const url of ["https://evilreddit.com/", "http://reddit.com/", "https://reddit.com@evil.net/"]) {
const result = await fetcher.execute(job(url));
assert.equal(result.type, "job_error");
assert.equal(result.code, REFUSALS.HOST_NOT_ALLOWED);
}
assert.equal(called, 0);
assert.equal(limits.inflight, 0);
});
test("body over the cap is truncated and flagged", async () => {
const big = "a".repeat(5000);
const { fetcher, limits } = build(async () => response(big));
limits.caps.max_body_bytes = 1000;
const result = await fetcher.execute(job());
assert.equal(result.type, "job_result");
assert.equal(result.truncated, true);
assert.equal(Buffer.from(result.body_b64, "base64").length, 1000);
});
test("the local bandwidth budget also bounds the body", async () => {
const { fetcher, usage } = build(async () => response("b".repeat(3 * 1024 * 1024)), { bandwidthMb: 1 });
const result = await fetcher.execute(job());
assert.equal(result.truncated, true);
assert.ok(Buffer.from(result.body_b64, "base64").length < 1024 * 1024);
assert.ok(usage.usedBytes <= 1024 * 1024);
});
test("exhausted bandwidth refuses before fetching", async () => {
let called = 0;
const { fetcher, usage } = build(
async () => {
called += 1;
return response("x");
},
{ bandwidthMb: 1 },
);
await usage.record({ ok: true, up: 0, down: 1024 * 1024 });
const result = await fetcher.execute(job());
assert.equal(result.code, REFUSALS.BANDWIDTH_EXHAUSTED);
assert.equal(called, 0);
});
test("concurrency cap refuses the extra job", async () => {
let release;
const gate = new Promise((resolve) => {
release = resolve;
});
const { fetcher } = build(
async () => {
await gate;
return response("x");
},
{ maxConcurrent: 1 },
);
const first = fetcher.execute(job());
const second = await fetcher.execute(job());
assert.equal(second.code, REFUSALS.CONCURRENCY_EXCEEDED);
release();
assert.equal((await first).type, "job_result");
});
test("timeout aborts the request and reports timeout", async () => {
const { fetcher, limits } = build(
(url, options) =>
new Promise((resolve, reject) => {
options.signal.addEventListener("abort", () => reject(new DOMException("aborted", "AbortError")));
}),
);
limits.caps.job_timeout_seconds = 1;
const started = Date.now();
const result = await fetcher.execute(job(undefined, { timeout: 1 }));
assert.equal(result.code, REFUSALS.TIMEOUT);
assert.ok(Date.now() - started < 3000);
assert.equal(limits.inflight, 0);
});
test("a redirect leaving the allowlist is refused and not followed", async () => {
const urls = [];
const { fetcher } = build(async (url) => {
urls.push(url);
return response("", { status: 302, headers: { location: "https://evil.net/steal" } });
});
const result = await fetcher.execute(job());
assert.equal(result.type, "job_error");
assert.equal(result.code, REFUSALS.REDIRECT_NOT_ALLOWED);
assert.equal(urls.length, 1);
});
test("an opaque redirect fails closed", async () => {
const opaque = { type: "opaqueredirect", status: 0, headers: new Headers(), body: null };
const { fetcher } = build(async () => opaque);
const result = await fetcher.execute(job());
assert.equal(result.code, REFUSALS.REDIRECT_NOT_ALLOWED);
});
test("a same-allowlist redirect is followed, with the policy checked on every hop", async () => {
const urls = [];
const { fetcher } = build(async (url) => {
urls.push(url);
if (urls.length === 1) {
return response("", { status: 301, headers: { location: "https://old.reddit.com/r/x" } });
}
return response("final");
});
const result = await fetcher.execute(job());
assert.equal(result.type, "job_result");
assert.equal(Buffer.from(result.body_b64, "base64").toString(), "final");
assert.deepEqual(urls, ["https://www.reddit.com/r/x.json", "https://old.reddit.com/r/x"]);
});
test("redirect loops are bounded", async () => {
let count = 0;
const { fetcher } = build(async () => {
count += 1;
return response("", { status: 302, headers: { location: "https://www.reddit.com/loop" } });
});
const result = await fetcher.execute(job());
assert.equal(result.code, REFUSALS.REDIRECT_NOT_ALLOWED);
assert.equal(count, MAX_REDIRECTS + 1);
});
test("a network failure is a loud job_error", async () => {
const { fetcher, usage } = build(async () => {
throw new TypeError("network down");
});
const result = await fetcher.execute(job());
assert.equal(result.type, "job_error");
assert.equal(result.code, "fetch_failed");
assert.equal(usage.snapshot().failed, 1);
});
nodes/extension/test/limits.test.mjs
+91
View File
@@ -0,0 +1,91 @@
import test from "node:test";
import assert from "node:assert/strict";
import { Limits } from "../src/Limits.js";
import { LIMITS, REFUSALS } from "../src/spec.js";
const MB = 1024 * 1024;
function at(hour, minute = 0) {
return () => new Date(2026, 0, 15, hour, minute);
}
test("defaults come from the baked spec", () => {
const limits = new Limits();
assert.equal(limits.concurrency, LIMITS.max_concurrent_per_node);
assert.equal(limits.bandwidthBytes, LIMITS.default_donor_mb_per_day * MB);
assert.equal(limits.maxBodyBytes, LIMITS.max_body_bytes);
});
test("local settings are clamped", () => {
const limits = new Limits({ local: { bandwidthMb: 10 ** 9, maxConcurrent: 99 } });
assert.equal(limits.local.bandwidthMb, LIMITS.max_donor_mb_per_day);
assert.equal(limits.local.maxConcurrent, LIMITS.max_concurrent_per_node);
limits.setLocal({ bandwidthMb: -5, maxConcurrent: 0 });
assert.equal(limits.local.bandwidthMb, LIMITS.default_donor_mb_per_day);
assert.equal(limits.local.maxConcurrent, LIMITS.max_concurrent_per_node);
limits.setLocal({ activeFrom: "25:99", activeTo: "10:00" });
assert.equal(limits.local.activeFrom, "");
});
test("a welcome can lower caps and never raise them", () => {
const limits = new Limits();
limits.applyWelcome(
{
max_body_bytes: 10 ** 9,
max_concurrent_per_node: 1,
job_timeout_seconds: 5,
ping_seconds: 999,
reconnect_min_seconds: 2,
reconnect_max_seconds: 10 ** 6,
},
{ bandwidth_mb_day: 5, max_concurrent: 99 },
);
assert.equal(limits.maxBodyBytes, LIMITS.max_body_bytes);
assert.equal(limits.concurrency, 1);
assert.equal(limits.timeoutSeconds(25), 5);
assert.equal(limits.pingSeconds, LIMITS.ping_seconds);
assert.equal(limits.bandwidthBytes, 5 * MB);
assert.equal(limits.reconnectMin, 2);
assert.equal(limits.reconnectMax, LIMITS.reconnect_max_seconds);
});
test("concurrency cap admits and releases", () => {
const limits = new Limits({ local: { maxConcurrent: 1 } });
assert.equal(limits.admit(0), null);
assert.equal(limits.admit(0), REFUSALS.CONCURRENCY_EXCEEDED);
limits.release();
assert.equal(limits.admit(0), null);
limits.release();
limits.release();
assert.equal(limits.inflight, 0);
});
test("bandwidth cap refuses when used up", () => {
const limits = new Limits({ local: { bandwidthMb: 1 } });
assert.equal(limits.admit(MB - 1), null);
limits.release();
assert.equal(limits.admit(MB), REFUSALS.BANDWIDTH_EXHAUSTED);
assert.equal(limits.remainingBytes(2 * MB), 0);
});
test("active hours, including a window that wraps midnight", () => {
const day = new Limits({ clock: at(12), local: { activeFrom: "09:00", activeTo: "17:00" } });
assert.equal(day.admit(0), null);
const early = new Limits({ clock: at(8, 59), local: { activeFrom: "09:00", activeTo: "17:00" } });
assert.equal(early.admit(0), REFUSALS.OUTSIDE_HOURS);
const end = new Limits({ clock: at(17), local: { activeFrom: "09:00", activeTo: "17:00" } });
assert.equal(end.admit(0), REFUSALS.OUTSIDE_HOURS);
const wrapIn = new Limits({ clock: at(23, 30), local: { activeFrom: "22:00", activeTo: "06:00" } });
assert.equal(wrapIn.inActiveHours(), true);
const wrapEarly = new Limits({ clock: at(3), local: { activeFrom: "22:00", activeTo: "06:00" } });
assert.equal(wrapEarly.inActiveHours(), true);
const wrapOut = new Limits({ clock: at(12), local: { activeFrom: "22:00", activeTo: "06:00" } });
assert.equal(wrapOut.inActiveHours(), false);
const always = new Limits({ clock: at(12) });
assert.equal(always.inActiveHours(), true);
});
test("hello limits advertise the effective concurrency", () => {
const limits = new Limits({ local: { maxConcurrent: 1 } });
assert.deepEqual(limits.helloLimits(), { max_concurrent: 1 });
});
nodes/extension/test/policy.test.mjs
+114
View File
@@ -0,0 +1,114 @@
import test from "node:test";
import assert from "node:assert/strict";
import { Policy } from "../src/Policy.js";
import { DESTINATIONS, REFUSALS } from "../src/spec.js";
const policy = new Policy();
test("allows https on the allowlisted hosts and subdomains", () => {
for (const url of [
"https://reddit.com/r/python.json",
"https://www.reddit.com/r/python.json",
"https://old.reddit.com/",
"https://redd.it/abc",
"https://REDDIT.COM./x",
]) {
assert.equal(policy.check("GET", url).ok, true, url);
}
});
test("suffix matching is dot safe", () => {
for (const url of [
"https://evilreddit.com/",
"https://reddit.com.evil.net/",
"https://notredd.it/",
"https://reddit.co/",
]) {
const verdict = policy.check("GET", url);
assert.equal(verdict.ok, false, url);
assert.equal(verdict.code, REFUSALS.HOST_NOT_ALLOWED);
}
});
test("userinfo tricks are refused", () => {
for (const url of [
"https://reddit.com@evil.net/",
"https://user:pass@reddit.com/",
"https://evil.net\\@reddit.com/",
]) {
assert.equal(policy.check("GET", url).ok, false, url);
}
});
test("non https schemes are refused", () => {
for (const url of ["http://reddit.com/", "ftp://reddit.com/", "ws://reddit.com/", "file:///etc/passwd"]) {
const verdict = policy.check("GET", url);
assert.equal(verdict.ok, false, url);
}
});
test("only port 443 passes over https", () => {
assert.equal(policy.check("GET", "https://reddit.com:443/").ok, true);
for (const url of ["https://reddit.com:80/", "https://reddit.com:8443/", "https://reddit.com:22/"]) {
const verdict = policy.check("GET", url);
assert.equal(verdict.ok, false, url);
assert.equal(verdict.code, REFUSALS.PORT_NOT_ALLOWED);
}
});
test("method must be permitted", () => {
const verdict = policy.check("POST", "https://reddit.com/");
assert.equal(verdict.code, REFUSALS.METHOD_NOT_ALLOWED);
assert.equal(policy.check("get", "https://reddit.com/").ok, true);
});
test("ip literals, junk and oversize urls are refused", () => {
for (const url of ["https://127.0.0.1/", "https://[::1]/", "https://10.0.0.1/", "nonsense", "", null, 5]) {
assert.equal(policy.check("GET", url).ok, false, String(url));
}
assert.equal(policy.check("GET", "https://reddit.com/" + "a".repeat(3000)).ok, false);
});
test("a redirect leaving the allowlist is refused with redirect_not_allowed", () => {
const base = new URL("https://www.reddit.com/r/x");
const bad = policy.checkRedirect("https://evil.net/landing", base, "GET");
assert.equal(bad.ok, false);
assert.equal(bad.code, REFUSALS.REDIRECT_NOT_ALLOWED);
assert.equal(policy.checkRedirect("http://reddit.com/x", base, "GET").code, REFUSALS.REDIRECT_NOT_ALLOWED);
assert.equal(policy.checkRedirect("//evilreddit.com/x", base, "GET").code, REFUSALS.REDIRECT_NOT_ALLOWED);
assert.equal(policy.checkRedirect("/r/y", base, "GET").ok, true);
assert.equal(policy.checkRedirect("https://old.reddit.com/r/y", base, "GET").ok, true);
});
test("a refreshed allowlist may only be equal or narrower", () => {
const narrower = [{ ...DESTINATIONS[0], suffix: "old.reddit.com" }];
const kept = Policy.narrow(DESTINATIONS, narrower);
assert.equal(kept.destinations.length, 1);
assert.equal(kept.rejected.length, 0);
const equal = Policy.narrow(DESTINATIONS, DESTINATIONS.map((item) => ({ ...item })));
assert.equal(equal.destinations.length, DESTINATIONS.length);
const widened = Policy.narrow(DESTINATIONS, [
{ suffix: "evil.net", adapter: "x", methods: ["GET"], ports: [443], modes: ["fetch"] },
{ suffix: "reddit.com", adapter: "reddit", methods: ["GET", "POST"], ports: [443], modes: ["fetch"] },
{ suffix: "reddit.com", adapter: "reddit", methods: ["GET"], ports: [443, 8080], modes: ["fetch"] },
{ suffix: "reddit.com", adapter: "reddit", methods: ["GET"], ports: [443], modes: ["tunnel"] },
{ suffix: "evilreddit.com", adapter: "reddit", methods: ["GET"], ports: [443], modes: ["fetch"] },
]);
assert.equal(widened.destinations.length, 0);
assert.equal(widened.rejected.length, 5);
const empty = Policy.narrow(DESTINATIONS, []);
assert.equal(empty.destinations.length, 0);
assert.equal(Policy.narrow(DESTINATIONS, "x").destinations.length, 0);
});
test("a narrowed policy refuses what the baked policy would allow", () => {
const { policy: narrow } = Policy.fromRefreshed(DESTINATIONS, [
{ ...DESTINATIONS[0], suffix: "old.reddit.com" },
]);
assert.equal(narrow.check("GET", "https://old.reddit.com/").ok, true);
assert.equal(narrow.check("GET", "https://www.reddit.com/").ok, false);
assert.equal(narrow.check("GET", "https://redd.it/x").ok, false);
});
nodes/extension/test/spec.test.mjs
+38
View File
@@ -0,0 +1,38 @@
import test from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { DESTINATIONS, LIMITS, REFUSALS, ALLOWED_PORTS, PROTOCOL_VERSION } from "../src/spec.js";
const source = readFileSync(
fileURLToPath(new URL("../../../devplacepy/services/nodes/spec.py", import.meta.url)),
"utf8",
);
function evaluate(expression) {
const clean = expression.replaceAll("_", "").trim();
assert.match(clean, /^[\d\s*]+$/, expression);
return clean.split("*").reduce((product, part) => product * Number(part), 1);
}
test("baked allowlist matches the server specification", () => {
const suffixes = [...source.matchAll(/DestinationSpec\("([^"]+)"/g)].map((match) => match[1]);
assert.deepEqual(DESTINATIONS.map((item) => item.suffix), suffixes);
});
test("baked ports, protocol and refusal codes match the server specification", () => {
const ports = /ALLOWED_PORTS = \(([^)]*)\)/.exec(source)[1].split(",").filter(Boolean).map(Number);
assert.deepEqual([...ALLOWED_PORTS], ports);
assert.equal(PROTOCOL_VERSION, Number(/PROTOCOL_VERSION = (\d+)/.exec(source)[1]));
const codes = [...source.matchAll(/RefusalSpec\("([a-z_]+)"/g)].map((match) => match[1]);
assert.deepEqual(Object.values(REFUSALS).sort(), codes.sort());
});
test("baked limits match the server LimitSpec defaults", () => {
for (const [key, value] of Object.entries(LIMITS)) {
const pattern = new RegExp(`^\\s+${key}: int = ([\\d_ *]+)$`, "m");
const match = pattern.exec(source);
assert.ok(match, `${key} missing from spec.py`);
assert.equal(value, evaluate(match[1]), key);
}
});
nodes/extension/test/wsclient.test.mjs
+323
View File
@@ -0,0 +1,323 @@
import test from "node:test";
import assert from "node:assert/strict";
import { Fetcher } from "../src/Fetcher.js";
import { Limits } from "../src/Limits.js";
import { Policy } from "../src/Policy.js";
import { State, STATES } from "../src/State.js";
import { Usage } from "../src/Usage.js";
import { WsClient } from "../src/WsClient.js";
import { DESTINATIONS } from "../src/spec.js";
function memoryArea() {
const store = {};
return {
async get(key) {
return key in store ? { [key]: store[key] } : {};
},
async set(values) {
Object.assign(store, values);
},
};
}
class FakeSocket {
static instances = [];
constructor(address) {
this.address = address;
this.readyState = 0;
this.sent = [];
this.closed = null;
FakeSocket.instances.push(this);
}
open() {
this.readyState = 1;
this.onopen();
}
send(text) {
this.sent.push(JSON.parse(text));
}
close(code, reason) {
this.readyState = 3;
this.closed = { code, reason };
}
receive(frame) {
return this.onmessage({ data: typeof frame === "string" ? frame : JSON.stringify(frame) });
}
serverClose(code, reason = "") {
this.readyState = 3;
return this.onclose({ code, reason });
}
}
class FakeTimers {
constructor() {
this.timeouts = [];
this.intervals = [];
}
setTimeout(fn, ms) {
const handle = { fn, ms, cleared: false };
this.timeouts.push(handle);
return handle;
}
clearTimeout(handle) {
handle.cleared = true;
}
setInterval(fn, ms) {
const handle = { fn, ms, cleared: false };
this.intervals.push(handle);
return handle;
}
clearInterval(handle) {
handle.cleared = true;
}
}
function build(overrides = {}) {
FakeSocket.instances = [];
const timers = new FakeTimers();
const limits = new Limits();
const usage = new Usage(memoryArea());
const state = new State(memoryArea());
const fetchLog = [];
const jobFetch = overrides.jobFetch || (async () => new Response("page", { status: 200 }));
const fetcher = new Fetcher({ policy: new Policy(), limits, usage, fetchImpl: jobFetch });
const ticketFetch =
overrides.ticketFetch ||
(async (url, options) => {
fetchLog.push({ url, options });
return new Response(JSON.stringify({ ticket: "tk1", expires_in: 30, ws_path: "/nodes/ws" }), {
status: 200,
});
});
const client = new WsClient({
config: { credentials: () => ({ serverUrl: "https://devplace.net", secret: "dpn_secretsecretsecret" }) },
limits,
fetcher,
state,
fetchImpl: ticketFetch,
WebSocketImpl: FakeSocket,
timers,
random: overrides.random || (() => 0.5),
clientVersion: "9.9.9",
platform: "firefox",
});
return { client, timers, state, fetcher, limits, usage, fetchLog };
}
const welcome = (extra = {}) => ({
type: "welcome",
device_uid: "dev1",
limits: { max_concurrent_per_node: 2, ping_seconds: 20 },
destinations: DESTINATIONS.map((item) => ({ ...item })),
settings: { bandwidth_mb_day: 500, max_concurrent: 2 },
...extra,
});
async function online(context) {
await context.client.start();
const socket = FakeSocket.instances.at(-1);
socket.open();
await socket.receive(welcome());
return socket;
}
test("start fetches a ticket with the bearer secret and opens the socket", async () => {
const context = build();
await context.client.start();
assert.equal(context.fetchLog[0].url, "https://devplace.net/nodes/ticket");
assert.equal(context.fetchLog[0].options.headers.Authorization, "Bearer dpn_secretsecretsecret");
assert.equal(context.fetchLog[0].options.credentials, "omit");
const socket = FakeSocket.instances[0];
assert.equal(socket.address, "wss://devplace.net/nodes/ws?ticket=tk1");
});
test("hello is sent on open with extension kind and fetch mode only", async () => {
const context = build();
await context.client.start();
const socket = FakeSocket.instances[0];
socket.open();
const hello = socket.sent[0];
assert.equal(hello.type, "hello");
assert.equal(hello.protocol, 1);
assert.equal(hello.kind, "extension");
assert.deepEqual(hello.modes, ["fetch"]);
assert.equal(hello.client_version, "9.9.9");
assert.equal(hello.platform, "firefox");
assert.deepEqual(hello.limits, { max_concurrent: 2 });
});
test("welcome puts the client online and starts the keepalive", async () => {
const context = build();
await online(context);
assert.equal(context.state.snapshot().state, STATES.ONLINE);
assert.equal(context.state.snapshot().deviceUid, "dev1");
assert.equal(context.timers.intervals[0].ms, 20000);
});
test("keepalive timer sends pong and a server ping is answered with pong", async () => {
const context = build();
const socket = await online(context);
context.timers.intervals[0].fn();
await socket.receive({ type: "ping" });
assert.deepEqual(
socket.sent.filter((frame) => frame.type === "pong"),
[{ type: "pong" }, { type: "pong" }],
);
});
test("a job is executed and answered with job_result then usage", async () => {
const context = build();
const socket = await online(context);
await socket.receive({
type: "job",
job_id: "j9",
method: "GET",
url: "https://www.reddit.com/r/x.json",
timeout: 5,
});
await context.client.idle();
const result = socket.sent.find((frame) => frame.type === "job_result");
assert.equal(result.job_id, "j9");
assert.equal(Buffer.from(result.body_b64, "base64").toString(), "page");
assert.equal(socket.sent.some((frame) => frame.type === "usage"), true);
});
test("a job for a forbidden host is answered with job_error", async () => {
const context = build();
const socket = await online(context);
await socket.receive({ type: "job", job_id: "j2", method: "GET", url: "https://evilreddit.com/", timeout: 5 });
await context.client.idle();
const error = socket.sent.find((frame) => frame.type === "job_error");
assert.deepEqual(error, { type: "job_error", job_id: "j2", code: "host_not_allowed" });
});
test("a welcome that widens the allowlist is ignored for the widened entries", async () => {
const context = build();
await context.client.start();
const socket = FakeSocket.instances[0];
socket.open();
await socket.receive(
welcome({
destinations: [
...DESTINATIONS.map((item) => ({ ...item })),
{ suffix: "evil.net", adapter: "x", methods: ["GET"], ports: [443], modes: ["fetch"] },
],
}),
);
assert.match(context.state.snapshot().lastError, /evil\.net/);
await socket.receive({ type: "job", job_id: "j3", method: "GET", url: "https://evil.net/", timeout: 5 });
await context.client.idle();
assert.equal(socket.sent.find((frame) => frame.type === "job_error").code, "host_not_allowed");
});
test("a job before welcome is a protocol fault and closes with 4400", async () => {
const context = build();
await context.client.start();
const socket = FakeSocket.instances[0];
socket.open();
await socket.receive({ type: "job", job_id: "j", method: "GET", url: "https://reddit.com/", timeout: 5 });
assert.equal(socket.closed.code, 4400);
assert.equal(context.state.snapshot().state, STATES.BACKOFF);
});
test("malformed frames are rejected loudly", async () => {
const context = build();
const socket = await online(context);
for (const bad of ["not json", "[]", JSON.stringify({ type: "nope" }), JSON.stringify({ type: "job", job_id: "x" })]) {
await assert.rejects(context.client.handleMessage(bad));
}
await assert.rejects(context.client.handleMessage(new ArrayBuffer(4)));
assert.equal(socket.sent.filter((frame) => frame.type === "job_result").length, 0);
});
test("kick halts without reconnecting", async () => {
const context = build();
const socket = await online(context);
await socket.receive({ type: "kick", reason: "replaced" });
assert.equal(context.state.snapshot().state, STATES.HALTED);
assert.match(context.state.snapshot().detail, /replaced/);
assert.equal(context.timers.timeouts.filter((handle) => !handle.cleared).length, 0);
await context.client.ensure();
assert.equal(FakeSocket.instances.length, 1);
});
test("a normal close schedules a full jitter reconnect within the bounds", async () => {
const context = build({ random: () => 0.999 });
const socket = await online(context);
await socket.serverClose(1006);
const timer = context.timers.timeouts.at(-1);
assert.ok(timer.ms >= 1000 && timer.ms <= 60000, String(timer.ms));
assert.equal(context.state.snapshot().state, STATES.BACKOFF);
timer.fn();
await new Promise((resolve) => setImmediate(resolve));
assert.equal(FakeSocket.instances.length, 2);
});
test("backoff grows with attempts and never exceeds reconnect_max", async () => {
const context = build({ random: () => 1 });
await context.client.start();
const delays = [];
for (let index = 0; index < 12; index += 1) {
const socket = FakeSocket.instances.at(-1);
await socket.serverClose(1006);
const timer = context.timers.timeouts.at(-1);
delays.push(timer.ms);
timer.fn();
await new Promise((resolve) => setImmediate(resolve));
}
assert.ok(delays[1] > delays[0]);
assert.ok(Math.max(...delays) <= 60000);
assert.equal(delays.at(-1), 60000);
});
test("retry owner close reconnects at the minimum delay", async () => {
const context = build({ random: () => 1 });
const socket = await online(context);
await socket.serverClose(4013);
assert.equal(context.timers.timeouts.at(-1).ms, 1000);
});
test("a consent or device refusal halts instead of hammering the server", async () => {
const context = build();
await context.client.start();
const socket = FakeSocket.instances[0];
await socket.serverClose(1008, "no_consent");
assert.equal(context.state.snapshot().state, STATES.HALTED);
assert.match(context.state.snapshot().detail, /no_consent/);
});
test("a ticket 401 halts, a ticket 500 retries", async () => {
const halted = build({
ticketFetch: async () => new Response(JSON.stringify({ error: "x" }), { status: 401 }),
});
await halted.client.start();
assert.equal(halted.state.snapshot().state, STATES.HALTED);
assert.equal(FakeSocket.instances.length, 0);
const retried = build({ ticketFetch: async () => new Response("boom", { status: 500 }) });
await retried.client.start();
assert.equal(retried.state.snapshot().state, STATES.BACKOFF);
});
test("stop closes the socket and clears timers", async () => {
const context = build();
const socket = await online(context);
await context.client.stop("user");
assert.equal(socket.closed.code, 1000);
assert.equal(context.timers.intervals[0].cleared, true);
assert.equal(context.state.snapshot().state, STATES.OFF);
});
test("settings_update can only lower the effective limits", async () => {
const context = build();
const socket = await online(context);
await socket.receive({ type: "settings_update", settings: { bandwidth_mb_day: 10, max_concurrent: 1 } });
assert.equal(context.limits.concurrency, 1);
assert.equal(context.limits.bandwidthBytes, 10 * 1024 * 1024);
});
nodes/extension/ui/disclosure.html
+55
View File
@@ -0,0 +1,55 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>DevPlace Node: what is shared</title>
<link rel="stylesheet" href="popup.css">
</head>
<body class="page">
<main>
<h1>Before you share anything</h1>
<p class="muted">DevPlace Node lets the community use a little of your internet connection to read public pages from sites that block data centers. Read this once. Nothing runs until you accept and switch it on.</p>
<section class="card">
<h2>What passes through your browser</h2>
<ul>
<li>The <strong>address of the page</strong> that is requested (host and full URL), for example a public Reddit thread.</li>
<li>The <strong>page content</strong> that comes back. It is read by this extension and sent to the DevPlace server.</li>
<li>Your <strong>public IP address</strong> is what the destination site sees. That is the point of donating.</li>
<li>How many requests and how many bytes you moved, counted on this device and on the server.</li>
</ul>
</section>
<section class="card">
<h2>What is never used</h2>
<ul>
<li>Your <strong>cookies, logins and accounts</strong>. Every request is sent without credentials, so you are never signed in as yourself.</li>
<li>Your browsing history, open tabs and other sites.</li>
<li>Any host outside the built-in list, any port other than the standard secure one, any address inside your own network.</li>
</ul>
</section>
<section class="card">
<h2>You stay in control</h2>
<ul>
<li>One on/off switch in the popup. Off means disconnected.</li>
<li>A daily data cap, a parallel request cap and optional active hours, enforced on this device even if the server asks for more.</li>
<li>Live counters of requests and bytes in the popup.</li>
<li>You can forget the device secret at any time, and revoke the device on your DevPlace profile.</li>
</ul>
<p class="muted">Pages fetched through your connection are public pages. Do not donate from a connection whose address you cannot afford to have associated with that traffic.</p>
</section>
<section class="card">
<div class="row">
<button type="button" class="primary" id="accept">I understand, allow it</button>
<button type="button" id="decline">Not now</button>
</div>
<p class="error" id="error" role="alert"></p>
<p class="muted" id="done"></p>
</section>
</main>
<script type="module" src="disclosure.js"></script>
</body>
</html>
nodes/extension/ui/disclosure.js
+31
View File
@@ -0,0 +1,31 @@
class Disclosure {
constructor(root) {
this.root = root;
this.$ = (id) => root.getElementById(id);
}
async accept() {
try {
const reply = await chrome.runtime.sendMessage({ cmd: "accept" });
if (!reply || reply.error) {
throw new Error(reply ? reply.error : "The background worker did not answer");
}
this.$("error").textContent = "";
this.$("done").textContent =
"Accepted. Open the extension popup, connect your device secret and switch sharing on.";
} catch (error) {
this.$("error").textContent = error.message;
}
}
decline() {
this.$("done").textContent = "Nothing was enabled. You can close this tab.";
}
start() {
this.$("accept").addEventListener("click", () => this.accept());
this.$("decline").addEventListener("click", () => this.decline());
}
}
new Disclosure(document).start();
nodes/extension/ui/popup.css
+226
View File
@@ -0,0 +1,226 @@
:root {
color-scheme: light dark;
--bg: #f6f7f9;
--card: #ffffff;
--text: #1c2128;
--muted: #5b6672;
--border: #d5dae0;
--accent: #2f6fed;
--accent-text: #ffffff;
--ok: #1a8a4a;
--warn: #b7791f;
--bad: #c0392b;
--radius: 8px;
--gap: 12px;
--font: system-ui, -apple-system, "Segoe UI", Roboto, sans-serif;
}
@media (prefers-color-scheme: dark) {
:root {
--bg: #14181d;
--card: #1d232a;
--text: #e8ecf0;
--muted: #97a3b0;
--border: #323b45;
--accent: #5b92ff;
--accent-text: #0b1220;
--ok: #4cc27f;
--warn: #e0b04a;
--bad: #ee7b6e;
}
}
* {
box-sizing: border-box;
}
body {
margin: 0;
background: var(--bg);
color: var(--text);
font: 14px/1.45 var(--font);
width: 100%;
min-width: 260px;
max-width: 420px;
}
body.page {
max-width: 640px;
margin: 0 auto;
}
main {
padding: var(--gap);
display: grid;
gap: var(--gap);
}
h1 {
font-size: 1.15rem;
margin: 0;
}
h2 {
font-size: 0.95rem;
margin: 0 0 6px;
}
p {
margin: 0 0 8px;
}
.card {
background: var(--card);
border: 1px solid var(--border);
border-radius: var(--radius);
padding: var(--gap);
}
.muted {
color: var(--muted);
font-size: 0.85rem;
}
.row {
display: flex;
flex-wrap: wrap;
align-items: center;
justify-content: space-between;
gap: 8px;
}
.status {
display: inline-flex;
align-items: center;
gap: 6px;
font-weight: 600;
}
.dot {
width: 10px;
height: 10px;
border-radius: 50%;
background: var(--muted);
}
.dot.online {
background: var(--ok);
}
.dot.connecting,
.dot.backoff {
background: var(--warn);
}
.dot.halted {
background: var(--bad);
}
.counters {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(110px, 1fr));
gap: 8px;
}
.counter {
border: 1px solid var(--border);
border-radius: var(--radius);
padding: 8px;
}
.counter strong {
display: block;
font-size: 1.1rem;
}
label {
display: grid;
gap: 4px;
margin-bottom: 8px;
font-size: 0.85rem;
color: var(--muted);
}
input {
width: 100%;
font: inherit;
color: var(--text);
background: var(--bg);
border: 1px solid var(--border);
border-radius: var(--radius);
padding: 7px 9px;
}
button,
.button {
font: inherit;
cursor: pointer;
color: var(--text);
background: var(--card);
border: 1px solid var(--border);
border-radius: var(--radius);
padding: 7px 12px;
text-decoration: none;
}
button.primary {
background: var(--accent);
color: var(--accent-text);
border-color: var(--accent);
}
button.danger {
color: var(--bad);
}
button:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.switch {
display: inline-flex;
align-items: center;
gap: 8px;
margin: 0;
color: var(--text);
font-size: 1rem;
font-weight: 600;
}
.switch input {
width: 20px;
height: 20px;
}
.notice {
border-left: 3px solid var(--warn);
padding-left: 10px;
}
.error {
color: var(--bad);
min-height: 1.2em;
word-break: break-word;
}
.pair {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(110px, 1fr));
gap: 8px;
}
details > summary {
cursor: pointer;
font-weight: 600;
margin-bottom: 8px;
}
ul {
margin: 0 0 8px;
padding-left: 20px;
}
li {
margin-bottom: 4px;
}
nodes/extension/ui/popup.html
+77
View File
@@ -0,0 +1,77 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>DevPlace Node</title>
<link rel="stylesheet" href="popup.css">
</head>
<body>
<main>
<div class="row">
<h1>DevPlace Node</h1>
<span class="status"><span class="dot" id="dot"></span><span id="state">off</span></span>
</div>
<section class="card" id="disclosure-card" hidden>
<p class="notice">Before this can run you must read what is shared.</p>
<a class="button primary" id="open-disclosure" href="disclosure.html" target="_blank" rel="noopener">Read the disclosure</a>
</section>
<section class="card">
<div class="row">
<label class="switch"><input type="checkbox" id="toggle"> <span id="toggle-label">Sharing off</span></label>
<span class="muted" id="label"></span>
</div>
<p class="muted" id="detail"></p>
<p class="error" id="error" role="alert"></p>
</section>
<section class="card">
<h2>Today</h2>
<div class="counters">
<div class="counter"><span class="muted">Requests</span><strong id="requests">0</strong></div>
<div class="counter"><span class="muted">Downloaded</span><strong id="down">0 B</strong></div>
<div class="counter"><span class="muted">Uploaded</span><strong id="up">0 B</strong></div>
<div class="counter"><span class="muted">Failed</span><strong id="failed">0</strong></div>
</div>
<p class="muted" id="lifetime"></p>
</section>
<section class="card">
<details id="settings-box">
<summary>Limits on this device</summary>
<form id="settings">
<div class="pair">
<label>Data per day (MB)<input type="number" id="bandwidth" min="1" step="1"></label>
<label>Parallel requests<input type="number" id="concurrency" min="1" step="1"></label>
</div>
<div class="pair">
<label>Active from<input type="time" id="from"></label>
<label>Active to<input type="time" id="to"></label>
</div>
<p class="muted">Leave the hours empty to share at any time. These limits are enforced here, whatever the server asks.</p>
<button type="submit">Save limits</button>
</form>
</details>
</section>
<section class="card">
<details id="connect-box">
<summary id="connect-summary">Connect to a server</summary>
<form id="connect">
<label>Server URL<input type="url" id="server" placeholder="https://devplace.net" required></label>
<label>Device secret<input type="password" id="secret" placeholder="dpn_..." autocomplete="off" required></label>
<label>Device label<input type="text" id="device-label" maxlength="64"></label>
<button type="submit" class="primary">Save and connect</button>
</form>
<p class="muted" id="saved"></p>
<button type="button" class="danger" id="forget" hidden>Forget this device</button>
</details>
</section>
<p class="muted"><a href="disclosure.html" target="_blank" rel="noopener">What is shared</a></p>
</main>
<script type="module" src="popup.js"></script>
</body>
</html>
nodes/extension/ui/popup.js
+155
View File
@@ -0,0 +1,155 @@
const POLL_MS = 2000;
const UNITS = ["B", "KB", "MB", "GB", "TB"];
const STATE_LABELS = {
off: "Off",
connecting: "Connecting",
online: "Online",
backoff: "Reconnecting",
halted: "Stopped",
};
function formatBytes(value) {
let size = Math.max(0, Number(value) || 0);
let unit = 0;
while (size >= 1024 && unit < UNITS.length - 1) {
size /= 1024;
unit += 1;
}
return `${unit === 0 ? size : size.toFixed(1)} ${UNITS[unit]}`;
}
class Popup {
constructor(root) {
this.root = root;
this.timer = null;
this.view = null;
this.$ = (id) => root.getElementById(id);
}
async send(message) {
const reply = await chrome.runtime.sendMessage(message);
if (!reply) {
throw new Error("The background worker did not answer");
}
if (reply.error) {
throw new Error(reply.error);
}
return reply;
}
showError(error) {
this.$("error").textContent = error ? error.message || String(error) : "";
}
async run(message) {
try {
this.render(await this.send(message));
this.showError(null);
} catch (error) {
this.showError(error);
}
}
render(view) {
this.view = view;
const { config, state, usage } = view;
this.$("dot").className = `dot ${state.state}`;
this.$("state").textContent = STATE_LABELS[state.state] || state.state;
this.$("detail").textContent = state.detail || state.lastError || "";
this.$("label").textContent = config.label;
this.$("disclosure-card").hidden = config.disclosed;
const toggle = this.$("toggle");
toggle.checked = config.enabled;
toggle.disabled = !config.disclosed || !config.hasSecret;
this.$("toggle-label").textContent = config.enabled ? "Sharing on" : "Sharing off";
this.$("requests").textContent = usage.requests;
this.$("failed").textContent = usage.failed;
this.$("down").textContent = formatBytes(usage.bytesDown);
this.$("up").textContent = formatBytes(usage.bytesUp);
this.$("lifetime").textContent =
`All time: ${usage.lifetimeRequests} requests, ${formatBytes(usage.lifetimeBytes)}`;
this.renderSettings(config);
this.renderConnection(config);
}
renderSettings(config) {
if (this.root.activeElement && this.$("settings").contains(this.root.activeElement)) {
return;
}
this.$("bandwidth").value = config.bandwidthMb;
this.$("concurrency").value = config.maxConcurrent;
this.$("from").value = config.activeFrom;
this.$("to").value = config.activeTo;
}
renderConnection(config) {
this.$("forget").hidden = !config.hasSecret;
this.$("connect-summary").textContent = config.hasSecret ? "Connection" : "Connect to a server";
this.$("saved").textContent = config.hasSecret
? `Connected to ${config.serverUrl} with secret ${config.secretPrefix}`
: "";
if (!config.hasSecret) {
this.$("connect-box").open = true;
}
}
async requestOrigin(serverUrl) {
const origin = new URL(serverUrl).origin;
const granted = await chrome.permissions.request({ origins: [origin + "/*"] });
if (!granted) {
throw new Error("Permission to talk to that server was not granted");
}
}
async onConnect(event) {
event.preventDefault();
const secretField = this.$("secret");
try {
await this.requestOrigin(this.$("server").value);
this.render(
await this.send({
cmd: "connect",
serverUrl: this.$("server").value,
secret: secretField.value,
label: this.$("device-label").value,
}),
);
secretField.value = "";
this.showError(null);
} catch (error) {
this.showError(error);
}
}
onSettings(event) {
event.preventDefault();
this.run({
cmd: "settings",
values: {
bandwidthMb: Number(this.$("bandwidth").value),
maxConcurrent: Number(this.$("concurrency").value),
activeFrom: this.$("from").value,
activeTo: this.$("to").value,
},
});
}
onForget() {
if (this.root.defaultView.confirm("Forget this device secret on this browser?")) {
this.run({ cmd: "forget" });
}
}
start() {
this.$("connect").addEventListener("submit", (event) => this.onConnect(event));
this.$("settings").addEventListener("submit", (event) => this.onSettings(event));
this.$("forget").addEventListener("click", () => this.onForget());
this.$("toggle").addEventListener("change", (event) =>
this.run({ cmd: "enable", enabled: event.target.checked }),
);
this.run({ cmd: "status" });
this.timer = setInterval(() => this.run({ cmd: "status" }), POLL_MS);
}
}
new Popup(document).start();