retoorandClaude Sonnet 5.5
02ab747cd0
DevPlace CI / test (push) Failing after 44m45s
Continuous Delivery could never be turned off once a crop was planted. Add a one-shot per-plot "stop after harvest" flag (POST /game/plot/stop, Devii tool game_plot_stop, game_plots.stop_after_harvest), cleared by the next plant, autoreplant claim or Refactor. Autoreplant now also respects crop locks through the shared _crop_lock predicate. Plan and measurements in happyplanting.md. Also includes the pending community sync, node network and related work in the tree; the full suite passed (4999) with all of it. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
152 lines
4.5 KiB
Python
152 lines
4.5 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
from devplacepy.services.nodes.circuit import CLOSED, HALF_OPEN, OPEN, CircuitBreaker
|
|
|
|
|
|
class Clock:
|
|
def __init__(self):
|
|
self.now = 0.0
|
|
|
|
def __call__(self):
|
|
return self.now
|
|
|
|
|
|
def make(failures=3, open_seconds=60):
|
|
clock = Clock()
|
|
return CircuitBreaker(failures, open_seconds, clock), clock
|
|
|
|
|
|
def test_unknown_keys_are_closed():
|
|
breaker, _ = make()
|
|
assert breaker.state("x") == CLOSED and breaker.allows("x")
|
|
|
|
|
|
def test_it_opens_at_the_threshold_and_not_before():
|
|
breaker, _ = make(failures=3)
|
|
breaker.failure("x")
|
|
breaker.failure("x")
|
|
assert breaker.allows("x")
|
|
breaker.failure("x")
|
|
assert breaker.state("x") == OPEN and not breaker.allows("x")
|
|
|
|
|
|
def test_a_success_resets_the_streak():
|
|
breaker, _ = make(failures=3)
|
|
breaker.failure("x")
|
|
breaker.failure("x")
|
|
breaker.success("x")
|
|
breaker.failure("x")
|
|
breaker.failure("x")
|
|
assert breaker.allows("x")
|
|
|
|
|
|
def test_it_goes_half_open_after_the_cooldown_and_admits_one_probe():
|
|
breaker, clock = make(failures=1, open_seconds=60)
|
|
breaker.failure("x")
|
|
clock.now = 59
|
|
assert breaker.state("x") == OPEN
|
|
clock.now = 60
|
|
assert breaker.state("x") == HALF_OPEN and breaker.allows("x")
|
|
breaker.begin("x")
|
|
assert not breaker.allows("x")
|
|
|
|
|
|
def test_a_failed_probe_reopens_for_a_full_cooldown():
|
|
breaker, clock = make(failures=1, open_seconds=60)
|
|
breaker.failure("x")
|
|
clock.now = 70
|
|
breaker.begin("x")
|
|
breaker.failure("x")
|
|
assert breaker.state("x") == OPEN
|
|
clock.now = 129
|
|
assert breaker.state("x") == OPEN
|
|
clock.now = 131
|
|
assert breaker.state("x") == HALF_OPEN
|
|
|
|
|
|
def test_a_successful_probe_closes_it():
|
|
breaker, clock = make(failures=1)
|
|
breaker.failure("x")
|
|
clock.now = 100
|
|
breaker.begin("x")
|
|
breaker.success("x")
|
|
assert breaker.state("x") == CLOSED and breaker.allows("x")
|
|
|
|
|
|
def test_keys_are_independent():
|
|
breaker, _ = make(failures=1)
|
|
breaker.failure(("node", "a"))
|
|
assert not breaker.allows(("node", "a"))
|
|
assert breaker.allows(("node", "b"))
|
|
assert breaker.allows(("destination", "a"))
|
|
|
|
|
|
def test_open_count_and_forget():
|
|
breaker, _ = make(failures=1)
|
|
breaker.failure("a")
|
|
breaker.failure("b")
|
|
assert breaker.open_count() == 2
|
|
breaker.forget("a")
|
|
assert breaker.open_count() == 1
|
|
|
|
|
|
def test_a_zero_threshold_still_needs_one_failure():
|
|
breaker = CircuitBreaker(failures=0)
|
|
assert breaker.allows("x")
|
|
breaker.failure("x")
|
|
assert not breaker.allows("x")
|
|
|
|
|
|
def test_snapshot_lists_open_circuits_with_time_left():
|
|
breaker, clock = make(failures=2, open_seconds=60)
|
|
breaker.failure(("destination", "reddit.com"))
|
|
breaker.failure(("destination", "reddit.com"))
|
|
clock.now = 20
|
|
rows = breaker.snapshot()
|
|
assert rows[0]["key"] == "destination:reddit.com" and rows[0]["state"] == "open"
|
|
assert 39 <= rows[0]["seconds_left"] <= 40 and rows[0]["failures"] == 2
|
|
|
|
|
|
def test_dirty_flag_follows_every_change():
|
|
breaker, _ = make(failures=2)
|
|
assert not breaker.dirty
|
|
breaker.failure("x")
|
|
assert breaker.dirty
|
|
breaker.dirty = False
|
|
breaker.success("x")
|
|
assert breaker.dirty
|
|
breaker.dirty = False
|
|
breaker.success("never-seen")
|
|
assert not breaker.dirty
|
|
|
|
|
|
def test_a_restart_keeps_an_open_circuit_open_until_its_time_is_up():
|
|
wall = {"now": 1_000_000.0}
|
|
clock = Clock()
|
|
first = CircuitBreaker(2, 60, clock, wall=lambda: wall["now"])
|
|
first.failure(("destination", "reddit.com"))
|
|
first.failure(("destination", "reddit.com"))
|
|
clock.now = 10
|
|
wall["now"] += 10
|
|
saved = first.snapshot()
|
|
second_clock = Clock()
|
|
second_clock.now = 5000
|
|
second = CircuitBreaker(2, 60, second_clock, wall=lambda: wall["now"] + 5)
|
|
assert second.restore(saved) == 1
|
|
assert second.state(("destination", "reddit.com")) == OPEN
|
|
second_clock.now += 60
|
|
assert second.state(("destination", "reddit.com")) == HALF_OPEN
|
|
|
|
|
|
def test_restore_ignores_closed_ancient_and_malformed_rows():
|
|
wall = {"now": 1_000_000.0}
|
|
breaker = CircuitBreaker(3, 60, Clock(), wall=lambda: wall["now"])
|
|
rows = [
|
|
{"key": "destination:a", "failures": 1, "opened_at": wall["now"]},
|
|
{"key": "destination:b", "failures": 3, "opened_at": wall["now"] - 10_000},
|
|
{"key": "nocolon", "failures": 9, "opened_at": wall["now"]},
|
|
{"key": "destination:c", "failures": 9, "opened_at": 0},
|
|
]
|
|
assert breaker.restore(rows) == 0
|
|
assert breaker.open_count() == 0
|