feat: implement MoloApp plans 01-15 with integration
Full C++23 framework tree: runtime, observability, security, data, app, realtime, supervisor, clients, CLI, example plus enterprise tenancy, audit, operability, governance, supplychain. Postgres via docker compose for tests.
This commit is contained in:
@@ -0,0 +1,334 @@
|
||||
# Part 13 operability: required base-tree edits
|
||||
|
||||
This file lists every edit the base tree needs to integrate the
|
||||
`src/operability/` part. Paths are repo-relative. Hunks are unified diffs
|
||||
anchored on the seam symbols; this worktree does not contain the base `src/`
|
||||
tree, so context line numbers are symbolic and the integrator anchors each
|
||||
hunk on the named symbol. No hunk changes existing behavior when the new
|
||||
code paths are not taken.
|
||||
|
||||
Conventions used by the new code:
|
||||
|
||||
- Includes use `-Isrc`: `"operability/..."`, `"supervisor/..."`, etc.
|
||||
- Namespaces are the leaf names from the seam spec: `molo::deploy`,
|
||||
`molo::coord`, `molo::resilience`, `molo::chaos`, `molo::debug`, with shared
|
||||
caps in `molo::operability::Caps` (`src/operability/caps.h`).
|
||||
- Production builds use `-Isrc` only. The `ADAPTER` shims under
|
||||
`src/operability/adapters/include/` are standalone-only (see
|
||||
`src/operability.mk`, `OPERABILITY_STANDALONE=1`) and must never be on the
|
||||
production include path.
|
||||
- All new logic takes `molo::common::Clock` and `molo::common::Observer` by
|
||||
reference for frozen-clock tests.
|
||||
- Static caps live in `src/operability/caps.h`. Nothing autotunes.
|
||||
|
||||
## 1. `Makefile`: include the operability fragment
|
||||
|
||||
Anchor: the per-part `.mk` include block (next to the other `-include`
|
||||
lines such as the supervisor fragment).
|
||||
|
||||
```diff
|
||||
--- a/Makefile
|
||||
+++ b/Makefile
|
||||
@@ -1,0 +1,1 @@
|
||||
-include src/cli.mk
|
||||
+-include src/operability.mk
|
||||
```
|
||||
|
||||
Effect: builds `build/liboperability.a` and the `operability`,
|
||||
`operability-tests`, `operability-check` targets.
|
||||
|
||||
## 2. `src/supervisor/common/kv_store.hpp`: fencing CAS extension
|
||||
|
||||
`molo::coord::Fence` and `molo::coord::DistLock` need compare-and-swap on
|
||||
(key, expected token). This adds one virtual plus the in-memory
|
||||
implementation; existing `get/set/erase/acquire/renew` signatures are
|
||||
untouched.
|
||||
|
||||
```diff
|
||||
--- a/src/supervisor/common/kv_store.hpp
|
||||
+++ b/src/supervisor/common/kv_store.hpp
|
||||
@@ class KvStore
|
||||
virtual bool acquire(const std::string& key, const std::string& value, std::int64_t ttl_ms) = 0;
|
||||
virtual bool renew(const std::string& key, const std::string& expected, std::int64_t ttl_ms) = 0;
|
||||
+ // Fencing extension: set key to next only when its current value equals
|
||||
+ // expected. Empty expected matches a missing key.
|
||||
+ virtual bool cas(const std::string& key, const std::string& expected, const std::string& next,
|
||||
+ std::int64_t ttl_ms) = 0;
|
||||
```
|
||||
|
||||
```diff
|
||||
--- a/src/supervisor/common/kv_store.hpp
|
||||
+++ b/src/supervisor/common/kv_store.hpp
|
||||
@@ class InMemoryKvStore
|
||||
bool renew(const std::string& key, const std::string& expected, std::int64_t ttl_ms) override;
|
||||
+ bool cas(const std::string& key, const std::string& expected, const std::string& next,
|
||||
+ std::int64_t ttl_ms) override;
|
||||
```
|
||||
|
||||
The `InMemoryKvStore::cas` body follows `renew` but stores `next` on match
|
||||
and inserts when the key is missing and `expected` is empty (mirror the
|
||||
ADAPTER shim at
|
||||
`src/operability/adapters/include/supervisor/common/kv_store.hpp`, which is
|
||||
the reference implementation). Every other `KvStore` backend must implement
|
||||
`cas` with the same semantics; a backend that cannot do atomic CAS must
|
||||
return false so callers treat fencing as unavailable.
|
||||
|
||||
## 3. `src/cli/ops/ops_routes.hpp`: debug handler declaration
|
||||
|
||||
Anchor: `molo::ops::OpsRoutes`, next to `healthz/readyz/startupz/metrics`.
|
||||
|
||||
```diff
|
||||
--- a/src/cli/ops/ops_routes.hpp
|
||||
+++ b/src/cli/ops/ops_routes.hpp
|
||||
@@ class OpsRoutes
|
||||
OpsResponse healthz() const;
|
||||
OpsResponse readyz() const;
|
||||
OpsResponse startupz() const;
|
||||
OpsResponse metrics(bool gzip) const;
|
||||
+ // Admin-only debug surface. Returns 404 unless the process was built with
|
||||
+ // MOLO_ENABLE_DEBUG and the caller is the admin listener (see the .cpp).
|
||||
+ OpsResponse debug(const std::string& path, const std::string& token, bool admin_listener) const;
|
||||
```
|
||||
|
||||
`OpsResponse` needs no change (`{status, content_type, body, gzip}` covers
|
||||
the debug responses; debug bodies are never gzipped by this handler).
|
||||
|
||||
## 4. `src/cli/ops/ops_routes.cpp`: debug handler definition
|
||||
|
||||
Anchor: after `OpsRoutes::metrics`. Delegates to `molo::debug::Endpoint`;
|
||||
public listeners always get 404, and builds without `MOLO_ENABLE_DEBUG`
|
||||
always get 404.
|
||||
|
||||
```diff
|
||||
--- a/src/cli/ops/ops_routes.cpp
|
||||
+++ b/src/cli/ops/ops_routes.cpp
|
||||
@@
|
||||
#include "cli/ops/ops_routes.hpp"
|
||||
+#include "operability/debug/endpoint.h"
|
||||
+#include "operability/debug/snapshot.h"
|
||||
@@ OpsResponse OpsRoutes::metrics(bool gzip) const
|
||||
...
|
||||
+OpsResponse OpsRoutes::debug(const std::string& path, const std::string& token,
|
||||
+ bool admin_listener) const {
|
||||
+ OpsResponse r;
|
||||
+ r.status = 404;
|
||||
+ r.content_type = "text/plain";
|
||||
+ r.body = "not found";
|
||||
+ r.gzip = false;
|
||||
+#ifdef MOLO_ENABLE_DEBUG
|
||||
+ if (!admin_listener) return r;
|
||||
+ // Auth gate: reuse the ops auth check used by the admin listener. The
|
||||
+ // predicate below is the integration point, see note in section 7.
|
||||
+ molo::debug::Endpoint ep([this] { return this->debug_snapshot(); },
|
||||
+ [](const std::string& t) { return ops_admin_auth(t); });
|
||||
+ const auto out = ep.handle(path, token);
|
||||
+ r.status = out.status;
|
||||
+ r.content_type = out.content_type;
|
||||
+ r.body = out.body;
|
||||
+#else
|
||||
+ (void)path;
|
||||
+ (void)token;
|
||||
+ (void)admin_listener;
|
||||
+#endif
|
||||
+ return r;
|
||||
+}
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- `debug_snapshot()` is a small new private helper on `OpsRoutes` (declare
|
||||
in the hpp next to `debug`) that builds a `molo::debug::Snapshot`,
|
||||
registers sections, and returns `capture()`. One suggested section per
|
||||
subsystem; keep each section small, the snapshot caps at 1 MB.
|
||||
- `ops_admin_auth` stands for the existing admin listener auth check (03
|
||||
security gate owner). Wire the real predicate; never accept an empty
|
||||
token.
|
||||
- Routing rule, to be enforced at the listener setup call site (exact mount
|
||||
file unconfirmed in the seam spec, see section 7): `GET /debug/*` routes
|
||||
to `OpsRoutes::debug` with `admin_listener=true` on the admin listener
|
||||
only. The public listener must not route `/debug/*` at all (404).
|
||||
|
||||
## 5. `src/supervisor/tasks/scheduler.hpp`: elect via coord
|
||||
|
||||
Anchor: `molo::tasks::Scheduler` private lease members. The scheduler keeps
|
||||
its public surface (`tick_once()`, `is_leader()`); only the mechanism moves
|
||||
onto `molo::coord::LeaderElection`.
|
||||
|
||||
```diff
|
||||
--- a/src/supervisor/tasks/scheduler.hpp
|
||||
+++ b/src/supervisor/tasks/scheduler.hpp
|
||||
@@
|
||||
#include "supervisor/tasks/scheduler.hpp"
|
||||
+#include "operability/coord/election.h"
|
||||
@@ class Scheduler
|
||||
private:
|
||||
- // old internal lease fields (leader key, ttl, renew deadline)
|
||||
+ molo::coord::LeaderElection election_;
|
||||
```
|
||||
|
||||
Construct `election_` with the existing queue/kv/clock/observer members:
|
||||
key `"molo/scheduler/leader"` (keep the current `Options::leader_key`
|
||||
default), ttl 10000, renew 3000, node id from the scheduler options or
|
||||
worker id.
|
||||
|
||||
## 6. `src/supervisor/tasks/scheduler.cpp`: delegate to the election
|
||||
|
||||
Anchor: `Scheduler::tick_once` and `Scheduler::is_leader`.
|
||||
|
||||
```diff
|
||||
--- a/src/supervisor/tasks/scheduler.cpp
|
||||
+++ b/src/supervisor/tasks/scheduler.cpp
|
||||
@@ bool Scheduler::tick_once()
|
||||
- // old elect_locked lease handling
|
||||
- ...
|
||||
+ const bool leader = election_.tick();
|
||||
+ if (!leader) return false; // follower: no dispatch this tick
|
||||
... // existing dispatch work, unchanged
|
||||
@@ bool Scheduler::is_leader() const
|
||||
- return lease_held_locked();
|
||||
+ return election_.is_leader();
|
||||
```
|
||||
|
||||
Delete the replaced private lease helpers. Observer metrics keep their
|
||||
existing names; the election additionally emits `election.won/lost` events
|
||||
and `election_renewed_total`.
|
||||
|
||||
## 7. `src/clients/request.h`: deadline header helper
|
||||
|
||||
Anchor: `molo::clients::ClientRequest`, next to `header()`.
|
||||
|
||||
```diff
|
||||
--- a/src/clients/request.h
|
||||
+++ b/src/clients/request.h
|
||||
@@ class ClientRequest
|
||||
// existing header/timeouts accessors
|
||||
+ // Absolute X-Deadline-Ms value when present and well formed.
|
||||
+ std::optional<std::int64_t> deadline_ms() const;
|
||||
```
|
||||
|
||||
Implement in the existing request source file with
|
||||
`molo::resilience::Deadline::parse_header` on the `X-Deadline-Ms` header:
|
||||
|
||||
```cpp
|
||||
std::optional<std::int64_t> ClientRequest::deadline_ms() const {
|
||||
const auto v = header("X-Deadline-Ms"); // existing accessor
|
||||
if (!v) return std::nullopt;
|
||||
return molo::resilience::Deadline::parse_header(*v);
|
||||
}
|
||||
```
|
||||
|
||||
(Adjust to the real `header()` return type: string, optional, or out-param.)
|
||||
|
||||
## 8. `src/clients/client.cpp`: breaker plus deadline propagation
|
||||
|
||||
Anchor: `molo::clients::Client::send`, before `send_inner`. Order:
|
||||
deadline outermost, then shedder/bulkhead/breaker, per the resilience
|
||||
wiring (exact middleware ordering slot in `src/app` is unconfirmed, see
|
||||
section 9; the client-side half below is exact).
|
||||
|
||||
```diff
|
||||
--- a/src/clients/client.cpp
|
||||
+++ b/src/clients/client.cpp
|
||||
@@
|
||||
#include "clients/client.h"
|
||||
+#include "operability/resilience/breaker.h"
|
||||
+#include "operability/resilience/deadline.h"
|
||||
@@ Task<ClientResponse> Client::send(ClientRequest req)
|
||||
+ // 1. Breaker gate, innermost stage: never retry into an open breaker.
|
||||
+ // `breaker_` is a per-host molo::resilience::Breaker owned by Client.
|
||||
+ if (!breaker_.allow()) {
|
||||
+ co_return ClientResponse::failure(503, "breaker open");
|
||||
+ }
|
||||
+ // 2. Deadline propagation: convert the absolute inbound X-Deadline-Ms to
|
||||
+ // remaining budget minus the 50 ms floor margin for the outbound hop.
|
||||
+ if (const auto abs = req.deadline_ms()) {
|
||||
+ molo::resilience::Deadline dl(clock_);
|
||||
+ const std::int64_t budget = dl.remaining_ms(*abs);
|
||||
+ if (budget == 0) {
|
||||
+ breaker_.on_failure();
|
||||
+ co_return ClientResponse::failure(504, "deadline exceeded");
|
||||
+ }
|
||||
+ req.set_header("X-Deadline-Ms",
|
||||
+ molo::resilience::Deadline::format_header(clock_.wall_ms() + budget));
|
||||
+ reqmutable_timeouts_for(budget); // clamp hop timeouts to budget
|
||||
+ }
|
||||
auto resp = co_await send_inner(std::move(req));
|
||||
+ if (resp.ok()) {
|
||||
+ breaker_.on_success();
|
||||
+ } else if (resp.status() >= 500 || resp.transport_error()) {
|
||||
+ breaker_.on_failure();
|
||||
+ }
|
||||
co_return resp;
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- `breaker_` needs a `molo::common::Clock` and `molo::common::Observer` at
|
||||
`Client` construction; add them to the existing client config or
|
||||
constructor the same way timeouts are threaded today.
|
||||
- `ClientResponse::failure`, `resp.ok()`, `resp.status()`,
|
||||
`resp.transport_error()` are placeholders for the real response API:
|
||||
count transport errors and 5xx as breaker failures, everything else as
|
||||
success. Never report a shed 503 from downstream as a local failure
|
||||
twice; the downstream already counted it.
|
||||
- Retry policy (`RetryPolicy::should_retry_status/backoff_for`) must skip
|
||||
retries while `breaker_.allow()` is false and must honor downstream
|
||||
`Retry-After` via `RetryPolicy::parse_retry_after`.
|
||||
- `reqmutable_timeouts_for` stands for clamping the outbound hop timeouts
|
||||
(`Timeouts{connect,read,write,pool_acquire,total}`) to the propagated
|
||||
budget. Keep `Deadline::budget_for(phase)` shares proportional.
|
||||
|
||||
## 9. Unconfirmed sites (from the seam spec, kept as gaps)
|
||||
|
||||
These need the base tree to pin the exact file; the hunks above are written
|
||||
so they apply once the site is known:
|
||||
|
||||
- Middleware ordering slot for deadline outermost then
|
||||
shedder/bulkhead/breaker in `src/app` (05 request routing). The
|
||||
per-request wiring is: `Deadline` check, `Shedder::should_shed` (503 plus
|
||||
`Retry-After` on shed), `Bulkhead::try_acquire` (shed on `kRejected`),
|
||||
then dispatch; `Breaker` stays innermost at the client (section 8).
|
||||
- Admin listener bind site that mounts `OpsRoutes` (section 4 routing
|
||||
rule). Public listener must 404 `/debug/*`.
|
||||
- Realtime close path (06, connection close 1001) and auth gate owner
|
||||
(03 security): `Lifecycle::run_drain` phase 2 should close realtime
|
||||
connections with 1001; the debug auth predicate must be the 03 gate.
|
||||
|
||||
## 10. CLI wiring (rollout, rollback, experiment, debug)
|
||||
|
||||
`molo::cli::Command(name, brief)` with `flag()/child()/handler()` is the
|
||||
seam (`src/cli/cli/command.hpp`). Register four commands at the existing
|
||||
command registry (same site that registers supervisor/task commands):
|
||||
|
||||
- `rollout start --strategy rolling|blue-green --batch N --max-unavailable M`
|
||||
driving `molo::deploy::Rolling` / `molo::deploy::BlueGreen` step loops
|
||||
with readiness probes polling `readyz/startupz`.
|
||||
- `rollout abort` and `rollback` driving `Strategy::abort` and
|
||||
`BlueGreen::rollback`.
|
||||
- `experiment start|abort --blast N --duration MS --max-burn X` driving
|
||||
`molo::chaos::Experiment` plus `molo::chaos::Injector` hook setup.
|
||||
- `debug snapshot` printing `molo::debug::Snapshot::capture()`.
|
||||
|
||||
Handlers return `ExitCode` and write through `Output&`; no new CLI
|
||||
framework code is needed. Exact registry file to be confirmed against the
|
||||
base tree.
|
||||
|
||||
## 11. Verification
|
||||
|
||||
Standalone (this worktree, no base tree):
|
||||
|
||||
```sh
|
||||
make -f src/operability.mk OPERABILITY_STANDALONE=1 operability-check
|
||||
```
|
||||
|
||||
This builds `build/liboperability.a` with
|
||||
`g++ -std=c++23 -O2 -Wall -Wextra -Wpedantic -Werror -pthread` and runs
|
||||
`build/operability_tests` (with `MOLO_ENABLE_DEBUG`) plus
|
||||
`build/operability_tests_nodebug` (without it). All eight suites
|
||||
(coord, lifecycle, breaker/bulkhead, shed/deadline, deploy, slo, chaos,
|
||||
debug) run on frozen clocks with no real waiting.
|
||||
|
||||
Production (base tree present): drop `OPERABILITY_STANDALONE=1` so only
|
||||
`-Isrc` is used and the real
|
||||
`supervisor/common/{clock,kv_store,observer}.hpp` headers satisfy the
|
||||
includes. The ADAPTER shims must not be on the include path.
|
||||
@@ -0,0 +1,59 @@
|
||||
// ADAPTER: standalone shim for missing base header "supervisor/common/clock.hpp".
|
||||
// Production build uses the real base header from the base tree; this file is
|
||||
// only on the include path for standalone verification (see operability.mk,
|
||||
// OPERABILITY_STANDALONE=1). Signatures mirror the seam spec.
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <thread>
|
||||
|
||||
namespace molo::common {
|
||||
|
||||
// Minimal clock abstraction. Production Clock has identical member names.
|
||||
class Clock {
|
||||
public:
|
||||
virtual ~Clock() = default;
|
||||
virtual std::int64_t wall_ms() const = 0;
|
||||
virtual std::int64_t steady_ms() const = 0;
|
||||
virtual void sleep_ms(std::int64_t ms) = 0;
|
||||
};
|
||||
|
||||
// Frozen clock for deterministic tests.
|
||||
class FrozenClock : public Clock {
|
||||
public:
|
||||
explicit FrozenClock(std::int64_t start_ms = 0) : now_(start_ms) {}
|
||||
|
||||
std::int64_t wall_ms() const override { return now_; }
|
||||
std::int64_t steady_ms() const override { return now_; }
|
||||
void sleep_ms(std::int64_t ms) override {
|
||||
if (ms > 0) now_ += ms;
|
||||
}
|
||||
void advance(std::int64_t ms) {
|
||||
if (ms > 0) now_ += ms;
|
||||
}
|
||||
void set(std::int64_t ms) { now_ = ms; }
|
||||
|
||||
private:
|
||||
std::int64_t now_;
|
||||
};
|
||||
|
||||
// System clock for non-test standalone runs.
|
||||
class SystemClock : public Clock {
|
||||
public:
|
||||
std::int64_t wall_ms() const override {
|
||||
return std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::system_clock::now().time_since_epoch())
|
||||
.count();
|
||||
}
|
||||
std::int64_t steady_ms() const override {
|
||||
return std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now().time_since_epoch())
|
||||
.count();
|
||||
}
|
||||
void sleep_ms(std::int64_t ms) override {
|
||||
if (ms > 0) std::this_thread::sleep_for(std::chrono::milliseconds(ms));
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace molo::common
|
||||
@@ -0,0 +1,124 @@
|
||||
// ADAPTER: standalone shim for missing base header "supervisor/common/kv_store.hpp".
|
||||
// Production build uses the real base header from the base tree; this file is
|
||||
// only on the include path for standalone verification (see operability.mk,
|
||||
// OPERABILITY_STANDALONE=1). Signatures mirror the seam spec. The cas() method
|
||||
// is the documented fencing extension (CAS on key plus expected token).
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
#include "supervisor/common/clock.hpp"
|
||||
|
||||
namespace molo::common {
|
||||
|
||||
// Minimal KV abstraction. TTLs are in milliseconds; ttl_ms <= 0 means no expiry.
|
||||
class KvStore {
|
||||
public:
|
||||
virtual ~KvStore() = default;
|
||||
virtual bool get(const std::string& key, std::string& value) = 0;
|
||||
virtual void set(const std::string& key, const std::string& value, std::int64_t ttl_ms) = 0;
|
||||
virtual void erase(const std::string& key) = 0;
|
||||
virtual bool acquire(const std::string& key, const std::string& value, std::int64_t ttl_ms) = 0;
|
||||
virtual bool renew(const std::string& key, const std::string& expected, std::int64_t ttl_ms) = 0;
|
||||
// Fencing extension: set only when current value equals expected.
|
||||
virtual bool cas(const std::string& key, const std::string& expected, const std::string& next,
|
||||
std::int64_t ttl_ms) = 0;
|
||||
};
|
||||
|
||||
// In-memory store with clock-driven expiry. Mirrors InMemoryKvStore(Clock&).
|
||||
class InMemoryKvStore : public KvStore {
|
||||
public:
|
||||
explicit InMemoryKvStore(Clock& clock) : clock_(clock) {}
|
||||
|
||||
bool get(const std::string& key, std::string& value) override {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
const std::int64_t now = clock_.steady_ms();
|
||||
auto it = map_.find(key);
|
||||
if (it == map_.end() || is_expired(it->second, now)) {
|
||||
if (it != map_.end()) map_.erase(it);
|
||||
return false;
|
||||
}
|
||||
value = it->second.value;
|
||||
return true;
|
||||
}
|
||||
|
||||
void set(const std::string& key, const std::string& value, std::int64_t ttl_ms) override {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
Entry e;
|
||||
e.value = value;
|
||||
e.expires_ms = ttl_ms > 0 ? clock_.steady_ms() + ttl_ms : 0;
|
||||
map_[key] = e;
|
||||
if (map_.size() > kMaxKeys) map_.erase(map_.begin());
|
||||
}
|
||||
|
||||
void erase(const std::string& key) override {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
map_.erase(key);
|
||||
}
|
||||
|
||||
bool acquire(const std::string& key, const std::string& value, std::int64_t ttl_ms) override {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
const std::int64_t now = clock_.steady_ms();
|
||||
auto it = map_.find(key);
|
||||
if (it != map_.end() && !is_expired(it->second, now)) return false;
|
||||
Entry e;
|
||||
e.value = value;
|
||||
e.expires_ms = ttl_ms > 0 ? now + ttl_ms : 0;
|
||||
map_[key] = e;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool renew(const std::string& key, const std::string& expected, std::int64_t ttl_ms) override {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
const std::int64_t now = clock_.steady_ms();
|
||||
auto it = map_.find(key);
|
||||
if (it == map_.end() || is_expired(it->second, now)) {
|
||||
if (it != map_.end()) map_.erase(it);
|
||||
return false;
|
||||
}
|
||||
if (it->second.value != expected) return false;
|
||||
it->second.expires_ms = ttl_ms > 0 ? now + ttl_ms : 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool cas(const std::string& key, const std::string& expected, const std::string& next,
|
||||
std::int64_t ttl_ms) override {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
const std::int64_t now = clock_.steady_ms();
|
||||
auto it = map_.find(key);
|
||||
if (it == map_.end() || is_expired(it->second, now)) {
|
||||
if (it != map_.end()) map_.erase(it);
|
||||
if (!expected.empty()) return false;
|
||||
Entry e;
|
||||
e.value = next;
|
||||
e.expires_ms = ttl_ms > 0 ? now + ttl_ms : 0;
|
||||
map_[key] = e;
|
||||
return true;
|
||||
}
|
||||
if (it->second.value != expected) return false;
|
||||
it->second.value = next;
|
||||
it->second.expires_ms = ttl_ms > 0 ? now + ttl_ms : 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
private:
|
||||
struct Entry {
|
||||
std::string value;
|
||||
std::int64_t expires_ms = 0; // 0 means no expiry
|
||||
};
|
||||
|
||||
static constexpr std::size_t kMaxKeys = 65536;
|
||||
|
||||
static bool is_expired(const Entry& e, std::int64_t now) {
|
||||
return e.expires_ms > 0 && e.expires_ms <= now;
|
||||
}
|
||||
|
||||
Clock& clock_;
|
||||
std::mutex mutex_;
|
||||
std::map<std::string, Entry> map_;
|
||||
};
|
||||
|
||||
} // namespace molo::common
|
||||
@@ -0,0 +1,80 @@
|
||||
// ADAPTER: standalone shim for missing base header "supervisor/common/observer.hpp".
|
||||
// Production build uses the real base header from the base tree; this file is
|
||||
// only on the include path for standalone verification (see operability.mk,
|
||||
// OPERABILITY_STANDALONE=1). Signatures mirror the seam spec.
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace molo::common {
|
||||
|
||||
// Minimal observer abstraction for transition and audit events.
|
||||
class Observer {
|
||||
public:
|
||||
virtual ~Observer() = default;
|
||||
virtual void event(const std::string& type, const std::string& detail) = 0;
|
||||
virtual void counter(const std::string& name, std::int64_t delta) = 0;
|
||||
virtual void gauge(const std::string& name, double value) = 0;
|
||||
};
|
||||
|
||||
// Recording observer for tests. Keeps every event up to a static cap.
|
||||
class RecordingObserver : public Observer {
|
||||
public:
|
||||
struct Record {
|
||||
std::string type;
|
||||
std::string detail;
|
||||
};
|
||||
|
||||
void event(const std::string& type, const std::string& detail) override {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (events_.size() >= kMaxEvents) return;
|
||||
events_.push_back(Record{type, detail});
|
||||
}
|
||||
|
||||
void counter(const std::string& name, std::int64_t delta) override {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
for (auto& c : counters_) {
|
||||
if (c.first == name) {
|
||||
c.second += delta;
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (counters_.size() < kMaxEvents) counters_.emplace_back(name, delta);
|
||||
}
|
||||
|
||||
void gauge(const std::string&, double) override {}
|
||||
|
||||
std::vector<Record> events() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return events_;
|
||||
}
|
||||
|
||||
std::int64_t counter_value(const std::string& name) const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
for (const auto& c : counters_) {
|
||||
if (c.first == name) return c.second;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
private:
|
||||
static constexpr std::size_t kMaxEvents = 4096;
|
||||
mutable std::mutex mutex_;
|
||||
std::vector<Record> events_;
|
||||
std::vector<std::pair<std::string, std::int64_t>> counters_;
|
||||
};
|
||||
|
||||
// Null observer for production wiring where no listener is attached.
|
||||
class NullObserver : public Observer {
|
||||
public:
|
||||
void event(const std::string&, const std::string&) override {}
|
||||
void counter(const std::string&, std::int64_t) override {}
|
||||
void gauge(const std::string&, double) override {}
|
||||
};
|
||||
|
||||
} // namespace molo::common
|
||||
@@ -0,0 +1,57 @@
|
||||
// Static ceilings for part 13 operability. Every limit is a compile-time
|
||||
// constant; nothing here autotunes. Production may only lower these values
|
||||
// through explicit config that clamps to the ceiling.
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
|
||||
namespace molo::operability {
|
||||
|
||||
// Single home for all operability caps. Never instantiated.
|
||||
class Caps {
|
||||
public:
|
||||
Caps() = delete;
|
||||
|
||||
// Coordination.
|
||||
static constexpr std::size_t kMaxLeases = 1024;
|
||||
static constexpr std::size_t kMaxLockWaiters = 256;
|
||||
static constexpr std::int64_t kLeaderTtlMs = 10000;
|
||||
static constexpr std::int64_t kLeaderRenewMs = 3000;
|
||||
static constexpr std::int64_t kMaxLeaseTtlMs = 3600000;
|
||||
|
||||
// Deployments.
|
||||
static constexpr std::size_t kMaxDeployInstances = 4096;
|
||||
static constexpr std::size_t kMaxDeployBatch = 256;
|
||||
static constexpr std::int64_t kDeployReadyTimeoutMs = 300000;
|
||||
static constexpr std::int64_t kBlueGreenRollbackWindowMs = 1800000;
|
||||
|
||||
// Graceful shutdown drain.
|
||||
static constexpr std::int64_t kDrainDeadlineMs = 30000;
|
||||
static constexpr std::int64_t kDrainPollMs = 50;
|
||||
|
||||
// Resilience.
|
||||
static constexpr int kBreakerConsecutiveFailures = 20;
|
||||
static constexpr int kBreakerWindowMinSamples = 10;
|
||||
static constexpr std::int64_t kBreakerWindowMs = 10000;
|
||||
static constexpr int kBreakerWindowFailurePct = 50;
|
||||
static constexpr int kBreakerHalfOpenProbes = 3;
|
||||
static constexpr std::int64_t kBreakerOpenHoldMs = 30000;
|
||||
static constexpr std::size_t kBulkheadMaxPartitions = 32;
|
||||
static constexpr std::size_t kBulkheadMaxConcurrent = 64;
|
||||
static constexpr std::size_t kBulkheadMaxQueue = 128;
|
||||
static constexpr std::int64_t kDeadlineFloorMarginMs = 50;
|
||||
static constexpr std::int64_t kShedRetryAfterMaxS = 120;
|
||||
static constexpr std::size_t kSloMaxRoutes = 1024;
|
||||
static constexpr std::size_t kSloMaxSamplesPerRoute = 8192;
|
||||
|
||||
// Chaos.
|
||||
static constexpr std::size_t kChaosMaxHooks = 64;
|
||||
static constexpr int kChaosMaxBlastPct = 100;
|
||||
|
||||
// Debug endpoints.
|
||||
static constexpr std::size_t kDebugMaxResponseBytes = 1048576;
|
||||
static constexpr std::size_t kDebugMaxPathBytes = 512;
|
||||
};
|
||||
|
||||
} // namespace molo::operability
|
||||
@@ -0,0 +1,87 @@
|
||||
#include "operability/chaos/experiment.h"
|
||||
|
||||
#include "operability/caps.h"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace molo::chaos {
|
||||
|
||||
Experiment::Experiment(::molo::common::Clock& clock, ::molo::common::Observer& observer,
|
||||
std::string name, int blast_pct, std::int64_t duration_ms, double max_burn)
|
||||
: clock_(clock),
|
||||
observer_(observer),
|
||||
name_(std::move(name)),
|
||||
blast_pct_(blast_pct),
|
||||
duration_ms_(duration_ms),
|
||||
max_burn_(max_burn) {
|
||||
if (blast_pct_ < 0) blast_pct_ = 0;
|
||||
if (blast_pct_ > ::molo::operability::Caps::kChaosMaxBlastPct) {
|
||||
blast_pct_ = ::molo::operability::Caps::kChaosMaxBlastPct;
|
||||
}
|
||||
if (duration_ms_ < 0) duration_ms_ = 0;
|
||||
if (max_burn_ < 0.0) max_burn_ = 0.0;
|
||||
}
|
||||
|
||||
bool Experiment::start() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (state_ != State::kIdle || name_.empty()) return false;
|
||||
state_ = State::kRunning;
|
||||
start_ms_ = clock_.steady_ms();
|
||||
observer_.event("chaos.experiment.start", name_);
|
||||
return true;
|
||||
}
|
||||
|
||||
void Experiment::finish() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (state_ != State::kRunning) return;
|
||||
state_ = State::kFinished;
|
||||
observer_.event("chaos.experiment.finish", name_);
|
||||
}
|
||||
|
||||
void Experiment::abort(const std::string& reason) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (state_ != State::kRunning) return;
|
||||
state_ = State::kAborted;
|
||||
observer_.event("chaos.experiment.abort", name_ + " " + reason);
|
||||
}
|
||||
|
||||
bool Experiment::should_apply(std::uint64_t seq) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (state_ != State::kRunning) return false;
|
||||
if (duration_ms_ > 0 && clock_.steady_ms() - start_ms_ >= duration_ms_) {
|
||||
state_ = State::kFinished;
|
||||
observer_.event("chaos.experiment.finish", name_);
|
||||
return false;
|
||||
}
|
||||
if (blast_pct_ <= 0) return false;
|
||||
if (blast_pct_ >= 100) return true;
|
||||
std::uint64_t x = seq + 0x9e3779b97f4a7c15ULL;
|
||||
x ^= x >> 30;
|
||||
x *= 0xbf58476d1ce4e5b9ULL;
|
||||
x ^= x >> 27;
|
||||
return (x % 100) < static_cast<std::uint64_t>(blast_pct_);
|
||||
}
|
||||
|
||||
bool Experiment::abort_if_breached(const BurnProbe& probe) {
|
||||
if (!probe) return false;
|
||||
double burn = 0.0;
|
||||
try {
|
||||
burn = probe();
|
||||
} catch (...) {
|
||||
return false;
|
||||
}
|
||||
if (burn <= max_burn_) return false;
|
||||
abort("burn ceiling breached");
|
||||
return true;
|
||||
}
|
||||
|
||||
Experiment::State Experiment::state() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return state_;
|
||||
}
|
||||
|
||||
const std::string& Experiment::name() const { return name_; }
|
||||
|
||||
int Experiment::blast_pct() const { return blast_pct_; }
|
||||
|
||||
} // namespace molo::chaos
|
||||
@@ -0,0 +1,56 @@
|
||||
// Guardrailed chaos experiment: bounds a set of hooks by blast radius,
|
||||
// duration and abort conditions. should_apply gates each operation by a
|
||||
// deterministic traffic sample; abort_if_breached stops the experiment
|
||||
// when the observed burn rate crosses the ceiling.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
namespace molo::common {
|
||||
class Clock;
|
||||
class Observer;
|
||||
} // namespace molo::common
|
||||
|
||||
namespace molo::chaos {
|
||||
|
||||
class Experiment {
|
||||
public:
|
||||
enum class State : int { kIdle = 0, kRunning = 1, kFinished = 2, kAborted = 3 };
|
||||
|
||||
using BurnProbe = std::function<double()>;
|
||||
|
||||
Experiment(::molo::common::Clock& clock, ::molo::common::Observer& observer, std::string name,
|
||||
int blast_pct, std::int64_t duration_ms, double max_burn);
|
||||
|
||||
Experiment(const Experiment&) = delete;
|
||||
Experiment& operator=(const Experiment&) = delete;
|
||||
|
||||
bool start();
|
||||
void finish();
|
||||
void abort(const std::string& reason);
|
||||
|
||||
// True when seq falls inside the blast radius while running.
|
||||
bool should_apply(std::uint64_t seq);
|
||||
// Check the burn probe; aborts when over the ceiling. True when aborted.
|
||||
bool abort_if_breached(const BurnProbe& probe);
|
||||
|
||||
State state() const;
|
||||
const std::string& name() const;
|
||||
int blast_pct() const;
|
||||
|
||||
private:
|
||||
::molo::common::Clock& clock_;
|
||||
::molo::common::Observer& observer_;
|
||||
std::string name_;
|
||||
int blast_pct_;
|
||||
std::int64_t duration_ms_;
|
||||
double max_burn_;
|
||||
mutable std::mutex mutex_;
|
||||
State state_ = State::kIdle;
|
||||
std::int64_t start_ms_ = 0;
|
||||
};
|
||||
|
||||
} // namespace molo::chaos
|
||||
@@ -0,0 +1,57 @@
|
||||
// Single chaos hook: a named fault point with a deterministic action.
|
||||
// Hooks are value objects; the Injector owns the registry (max 64) and
|
||||
// decides when each hook fires. Header only.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
namespace molo::chaos {
|
||||
|
||||
class Hook {
|
||||
public:
|
||||
enum class Action : int {
|
||||
kNone = 0,
|
||||
kDelay = 1, // sleep delay_ms before continuing
|
||||
kFail = 2, // return an injected failure
|
||||
kAbort = 3 // abort the operation outright
|
||||
};
|
||||
|
||||
Hook() = default;
|
||||
Hook(std::string name, Action action, int fire_pct = 100, std::int64_t delay_ms = 0)
|
||||
: name_(std::move(name)), action_(action), fire_pct_(fire_pct), delay_ms_(delay_ms) {
|
||||
if (fire_pct_ < 0) fire_pct_ = 0;
|
||||
if (fire_pct_ > 100) fire_pct_ = 100;
|
||||
if (delay_ms_ < 0) delay_ms_ = 0;
|
||||
}
|
||||
|
||||
const std::string& name() const { return name_; }
|
||||
Action action() const { return action_; }
|
||||
int fire_pct() const { return fire_pct_; }
|
||||
std::int64_t delay_ms() const { return delay_ms_; }
|
||||
|
||||
// Deterministic sampler: hashes the sequence number so tests can predict
|
||||
// exactly which calls fire without any randomness.
|
||||
bool fires_for(std::uint64_t seq) const {
|
||||
if (action_ == Action::kNone || fire_pct_ <= 0) return false;
|
||||
if (fire_pct_ >= 100) return true;
|
||||
return (mix(seq) % 100) < static_cast<std::uint64_t>(fire_pct_);
|
||||
}
|
||||
|
||||
private:
|
||||
static std::uint64_t mix(std::uint64_t x) {
|
||||
x ^= x >> 33;
|
||||
x *= 0xff51afd7ed558ccdULL;
|
||||
x ^= x >> 33;
|
||||
x *= 0xc4ceb9fe1a85ec53ULL;
|
||||
x ^= x >> 33;
|
||||
return x;
|
||||
}
|
||||
|
||||
std::string name_;
|
||||
Action action_ = Action::kNone;
|
||||
int fire_pct_ = 100;
|
||||
std::int64_t delay_ms_ = 0;
|
||||
};
|
||||
|
||||
} // namespace molo::chaos
|
||||
@@ -0,0 +1,86 @@
|
||||
#include "operability/chaos/injector.h"
|
||||
|
||||
#include "operability/caps.h"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace molo::chaos {
|
||||
|
||||
Injector::Injector(::molo::common::Clock& clock, ::molo::common::Observer& observer)
|
||||
: clock_(clock), observer_(observer) {
|
||||
static_assert(64 >= 1, "hook table must be non-empty");
|
||||
if (::molo::operability::Caps::kChaosMaxHooks < 64) {
|
||||
// Static table sized to the cap; see header.
|
||||
}
|
||||
}
|
||||
|
||||
bool Injector::add_hook(Hook hook) {
|
||||
if (hook.name().empty()) return false;
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
for (std::size_t i = 0; i < count_; ++i) {
|
||||
if (hooks_[i].name() == hook.name()) return false;
|
||||
}
|
||||
if (count_ >= ::molo::operability::Caps::kChaosMaxHooks || count_ >= hooks_.size()) {
|
||||
return false;
|
||||
}
|
||||
hooks_[count_++] = std::move(hook);
|
||||
observer_.counter("chaos_hooks_total", 1);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Injector::remove_hook(const std::string& name) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
for (std::size_t i = 0; i < count_; ++i) {
|
||||
if (hooks_[i].name() == name) {
|
||||
for (std::size_t j = i + 1; j < count_; ++j) hooks_[j - 1] = hooks_[j];
|
||||
--count_;
|
||||
observer_.counter("chaos_hooks_removed_total", 1);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void Injector::clear() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
count_ = 0;
|
||||
}
|
||||
|
||||
std::size_t Injector::size() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return count_;
|
||||
}
|
||||
|
||||
Injector::Decision Injector::evaluate(std::uint64_t seq) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
Decision worst = Decision::kProceed;
|
||||
for (std::size_t i = 0; i < count_; ++i) {
|
||||
const Hook& h = hooks_[i];
|
||||
if (!h.fires_for(seq + i)) continue;
|
||||
switch (h.action()) {
|
||||
case Hook::Action::kNone:
|
||||
break;
|
||||
case Hook::Action::kDelay:
|
||||
clock_.sleep_ms(h.delay_ms());
|
||||
observer_.counter("chaos_delayed_total", 1);
|
||||
if (worst == Decision::kProceed) worst = Decision::kDelayed;
|
||||
break;
|
||||
case Hook::Action::kFail:
|
||||
observer_.event("chaos.fired", h.name());
|
||||
observer_.counter("chaos_failed_total", 1);
|
||||
if (worst == Decision::kProceed || worst == Decision::kDelayed) {
|
||||
worst = Decision::kFailed;
|
||||
}
|
||||
break;
|
||||
case Hook::Action::kAbort:
|
||||
observer_.event("chaos.fired", h.name());
|
||||
observer_.counter("chaos_aborted_total", 1);
|
||||
worst = Decision::kAborted;
|
||||
break;
|
||||
}
|
||||
if (worst == Decision::kAborted) break;
|
||||
}
|
||||
return worst;
|
||||
}
|
||||
|
||||
} // namespace molo::chaos
|
||||
@@ -0,0 +1,46 @@
|
||||
// Fault injector: owns up to 64 hooks and evaluates them on the request
|
||||
// path. evaluate() is the single choke point: production calls it from
|
||||
// middleware before dispatch, tests call it directly with a sequence id.
|
||||
#pragma once
|
||||
|
||||
#include <array>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
#include "operability/chaos/hook.h"
|
||||
|
||||
namespace molo::common {
|
||||
class Clock;
|
||||
class Observer;
|
||||
} // namespace molo::common
|
||||
|
||||
namespace molo::chaos {
|
||||
|
||||
class Injector {
|
||||
public:
|
||||
enum class Decision : int { kProceed = 0, kDelayed = 1, kFailed = 2, kAborted = 3 };
|
||||
|
||||
Injector(::molo::common::Clock& clock, ::molo::common::Observer& observer);
|
||||
|
||||
Injector(const Injector&) = delete;
|
||||
Injector& operator=(const Injector&) = delete;
|
||||
|
||||
bool add_hook(Hook hook);
|
||||
bool remove_hook(const std::string& name);
|
||||
void clear();
|
||||
std::size_t size() const;
|
||||
|
||||
// Evaluate all hooks for one operation. seq drives deterministic sampling.
|
||||
Decision evaluate(std::uint64_t seq);
|
||||
|
||||
private:
|
||||
::molo::common::Clock& clock_;
|
||||
::molo::common::Observer& observer_;
|
||||
mutable std::mutex mutex_;
|
||||
std::array<Hook, 64> hooks_;
|
||||
std::size_t count_ = 0;
|
||||
};
|
||||
|
||||
} // namespace molo::chaos
|
||||
@@ -0,0 +1,63 @@
|
||||
#include "operability/coord/dist_lock.h"
|
||||
|
||||
#include "operability/caps.h"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/kv_store.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace molo::coord {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr std::int64_t kPollMs = 25;
|
||||
|
||||
} // namespace
|
||||
|
||||
DistLock::DistLock(::molo::common::KvStore& kv, ::molo::common::Clock& clock,
|
||||
::molo::common::Observer& observer, std::string key, std::string owner)
|
||||
: kv_(kv), clock_(clock), observer_(observer), fence_(kv, key), key_(key), owner_(owner) {}
|
||||
|
||||
std::uint64_t DistLock::try_lock(std::int64_t ttl_ms, std::int64_t wait_ms) {
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (locked_) return 0;
|
||||
}
|
||||
if (key_.empty() || owner_.empty() || ttl_ms <= 0) return 0;
|
||||
if (wait_ms < 0) wait_ms = 0;
|
||||
if (ttl_ms > ::molo::operability::Caps::kMaxLeaseTtlMs) {
|
||||
ttl_ms = ::molo::operability::Caps::kMaxLeaseTtlMs;
|
||||
}
|
||||
const std::int64_t deadline = clock_.steady_ms() + wait_ms;
|
||||
for (;;) {
|
||||
if (kv_.acquire_ms(key_, owner_, ttl_ms)) {
|
||||
const std::uint64_t token = fence_.next(ttl_ms);
|
||||
if (token == 0) {
|
||||
kv_.erase(key_);
|
||||
return 0;
|
||||
}
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
locked_ = true;
|
||||
observer_.event("lock.acquired", key_);
|
||||
observer_.counter("lock_acquired_total", 1);
|
||||
return token;
|
||||
}
|
||||
if (clock_.steady_ms() >= deadline) return 0;
|
||||
clock_.sleep_ms(kPollMs);
|
||||
}
|
||||
}
|
||||
|
||||
void DistLock::unlock() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (!locked_) return;
|
||||
locked_ = false;
|
||||
std::string cur;
|
||||
if (kv_.get(key_, cur) && cur == owner_) kv_.erase(key_);
|
||||
observer_.event("lock.released", key_);
|
||||
}
|
||||
|
||||
bool DistLock::locked() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return locked_;
|
||||
}
|
||||
|
||||
} // namespace molo::coord
|
||||
@@ -0,0 +1,44 @@
|
||||
// Distributed mutex built on Lease. try_lock polls until the lease is held
|
||||
// or the wait budget expires; on success it returns a fencing token from
|
||||
// the key scoped fence so guarded writes can reject stale holders.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
#include "operability/coord/fence.h"
|
||||
|
||||
namespace molo::common {
|
||||
class Clock;
|
||||
class KvStore;
|
||||
class Observer;
|
||||
} // namespace molo::common
|
||||
|
||||
namespace molo::coord {
|
||||
|
||||
class DistLock {
|
||||
public:
|
||||
DistLock(::molo::common::KvStore& kv, ::molo::common::Clock& clock,
|
||||
::molo::common::Observer& observer, std::string key, std::string owner);
|
||||
|
||||
DistLock(const DistLock&) = delete;
|
||||
DistLock& operator=(const DistLock&) = delete;
|
||||
|
||||
// Returns the fencing token on success, 0 on timeout or error.
|
||||
std::uint64_t try_lock(std::int64_t ttl_ms, std::int64_t wait_ms);
|
||||
void unlock();
|
||||
bool locked() const;
|
||||
|
||||
private:
|
||||
::molo::common::KvStore& kv_;
|
||||
::molo::common::Clock& clock_;
|
||||
::molo::common::Observer& observer_;
|
||||
Fence fence_;
|
||||
std::string key_;
|
||||
std::string owner_;
|
||||
mutable std::mutex mutex_;
|
||||
bool locked_ = false;
|
||||
};
|
||||
|
||||
} // namespace molo::coord
|
||||
@@ -0,0 +1,74 @@
|
||||
#include "operability/coord/election.h"
|
||||
|
||||
#include "operability/caps.h"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/kv_store.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace molo::coord {
|
||||
|
||||
LeaderElection::LeaderElection(::molo::common::KvStore& kv, ::molo::common::Clock& clock,
|
||||
::molo::common::Observer& observer, std::string key,
|
||||
std::string node_id, std::int64_t ttl_ms, std::int64_t renew_ms)
|
||||
: kv_(kv),
|
||||
clock_(clock),
|
||||
observer_(observer),
|
||||
key_(std::move(key)),
|
||||
node_id_(std::move(node_id)),
|
||||
ttl_ms_(ttl_ms),
|
||||
renew_ms_(renew_ms) {
|
||||
if (ttl_ms_ <= 0) ttl_ms_ = ::molo::operability::Caps::kLeaderTtlMs;
|
||||
if (ttl_ms_ > ::molo::operability::Caps::kMaxLeaseTtlMs) {
|
||||
ttl_ms_ = ::molo::operability::Caps::kMaxLeaseTtlMs;
|
||||
}
|
||||
if (renew_ms_ <= 0 || renew_ms_ >= ttl_ms_) renew_ms_ = ttl_ms_ / 3;
|
||||
if (renew_ms_ <= 0) renew_ms_ = 1;
|
||||
}
|
||||
|
||||
bool LeaderElection::tick() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (key_.empty() || node_id_.empty()) return false;
|
||||
const std::int64_t now = clock_.steady_ms();
|
||||
if (leader_) {
|
||||
if (now - last_action_ms_ < renew_ms_) return true;
|
||||
if (kv_.renew_ms(key_, node_id_, ttl_ms_)) {
|
||||
last_action_ms_ = now;
|
||||
observer_.counter("election_renewed_total", 1);
|
||||
return true;
|
||||
}
|
||||
leader_ = false;
|
||||
observer_.event("election.lost", node_id_);
|
||||
return false;
|
||||
}
|
||||
if (kv_.acquire_ms(key_, node_id_, ttl_ms_)) {
|
||||
leader_ = true;
|
||||
last_action_ms_ = now;
|
||||
observer_.event("election.won", node_id_);
|
||||
observer_.counter("election_won_total", 1);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool LeaderElection::is_leader() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return leader_;
|
||||
}
|
||||
|
||||
std::string LeaderElection::leader() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
std::string cur;
|
||||
if (kv_.get(key_, cur)) return cur;
|
||||
return {};
|
||||
}
|
||||
|
||||
void LeaderElection::resign() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (!leader_) return;
|
||||
leader_ = false;
|
||||
std::string cur;
|
||||
if (kv_.get(key_, cur) && cur == node_id_) kv_.erase(key_);
|
||||
observer_.event("election.resigned", node_id_);
|
||||
}
|
||||
|
||||
} // namespace molo::coord
|
||||
@@ -0,0 +1,47 @@
|
||||
// Leader election on top of the KV store lease. The caller drives tick()
|
||||
// from the event loop (or a timer); tick acquires the leader key when free
|
||||
// and renews it while leader. Transitions are reported via the observer.
|
||||
// The task scheduler migrates its internal lease onto this class.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
namespace molo::common {
|
||||
class Clock;
|
||||
class KvStore;
|
||||
class Observer;
|
||||
} // namespace molo::common
|
||||
|
||||
namespace molo::coord {
|
||||
|
||||
class LeaderElection {
|
||||
public:
|
||||
LeaderElection(::molo::common::KvStore& kv, ::molo::common::Clock& clock,
|
||||
::molo::common::Observer& observer, std::string key, std::string node_id,
|
||||
std::int64_t ttl_ms = 10000, std::int64_t renew_ms = 3000);
|
||||
|
||||
LeaderElection(const LeaderElection&) = delete;
|
||||
LeaderElection& operator=(const LeaderElection&) = delete;
|
||||
|
||||
// One election step. Returns true when this node is leader afterwards.
|
||||
bool tick();
|
||||
bool is_leader() const;
|
||||
std::string leader() const;
|
||||
void resign();
|
||||
|
||||
private:
|
||||
::molo::common::KvStore& kv_;
|
||||
::molo::common::Clock& clock_;
|
||||
::molo::common::Observer& observer_;
|
||||
std::string key_;
|
||||
std::string node_id_;
|
||||
std::int64_t ttl_ms_;
|
||||
std::int64_t renew_ms_;
|
||||
mutable std::mutex mutex_;
|
||||
bool leader_ = false;
|
||||
std::int64_t last_action_ms_ = 0;
|
||||
};
|
||||
|
||||
} // namespace molo::coord
|
||||
@@ -0,0 +1,77 @@
|
||||
// Fencing token guard. A fenced resource only accepts work carrying the
|
||||
// newest token, so a stale lock holder cannot corrupt state after its lease
|
||||
// expired. Tokens are monotonic per Fence; the distributed value lives in
|
||||
// the KV store under the lock key and is advanced by CAS. Header only.
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
#include "supervisor/common/kv_store.hpp"
|
||||
|
||||
namespace molo::coord {
|
||||
|
||||
class Fence {
|
||||
public:
|
||||
explicit Fence(::molo::common::KvStore& kv, std::string key)
|
||||
: kv_(kv), key_(std::move(key)), local_(0) {}
|
||||
|
||||
Fence(const Fence&) = delete;
|
||||
Fence& operator=(const Fence&) = delete;
|
||||
|
||||
// Advance the fence and return the new token. Uses CAS so concurrent
|
||||
// advancers serialize; each caller gets a distinct increasing token.
|
||||
std::uint64_t next(std::int64_t ttl_ms = 60000) {
|
||||
for (int attempt = 0; attempt < 16; ++attempt) {
|
||||
std::string cur;
|
||||
const bool have = kv_.get(fence_key(), cur);
|
||||
const std::uint64_t cur_token = have ? parse(cur) : 0;
|
||||
std::uint64_t want = cur_token + 1;
|
||||
const std::uint64_t local = local_.load();
|
||||
if (want <= local) want = local + 1;
|
||||
const std::string next_val = std::to_string(want);
|
||||
const bool ok = have ? kv_.cas_ms(fence_key(), cur, next_val, ttl_ms)
|
||||
: kv_.acquire_ms(fence_key(), next_val, ttl_ms);
|
||||
if (ok) {
|
||||
std::uint64_t expect = local;
|
||||
while (expect < want && !local_.compare_exchange_weak(expect, want)) {
|
||||
}
|
||||
return want;
|
||||
}
|
||||
}
|
||||
return 0; // contention: caller treats 0 as failure
|
||||
}
|
||||
|
||||
// Returns true when the token is still the newest known token.
|
||||
bool check(std::uint64_t token) {
|
||||
std::string cur;
|
||||
if (!kv_.get(fence_key(), cur)) return token >= local_.load();
|
||||
const std::uint64_t remote = parse(cur);
|
||||
std::uint64_t expect = local_.load();
|
||||
while (expect < remote && !local_.compare_exchange_weak(expect, remote)) {
|
||||
}
|
||||
return token >= remote;
|
||||
}
|
||||
|
||||
const std::string& key() const { return key_; }
|
||||
|
||||
private:
|
||||
static std::uint64_t parse(const std::string& s) {
|
||||
std::uint64_t v = 0;
|
||||
for (char c : s) {
|
||||
if (c < '0' || c > '9') break;
|
||||
v = v * 10 + static_cast<std::uint64_t>(c - '0');
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
std::string fence_key() const { return key_ + "/fence"; }
|
||||
|
||||
::molo::common::KvStore& kv_;
|
||||
std::string key_;
|
||||
std::atomic<std::uint64_t> local_;
|
||||
};
|
||||
|
||||
} // namespace molo::coord
|
||||
@@ -0,0 +1,74 @@
|
||||
#include "operability/coord/lease.h"
|
||||
|
||||
#include "operability/caps.h"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/kv_store.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace molo::coord {
|
||||
|
||||
std::atomic<std::size_t> Lease::live_{0};
|
||||
|
||||
Lease::Lease(::molo::common::KvStore& kv, ::molo::common::Clock& clock,
|
||||
::molo::common::Observer& observer, std::string key, std::string owner)
|
||||
: kv_(kv), clock_(clock), observer_(observer), key_(std::move(key)), owner_(std::move(owner)) {
|
||||
live_.fetch_add(1);
|
||||
}
|
||||
|
||||
Lease::~Lease() {
|
||||
release();
|
||||
live_.fetch_sub(1);
|
||||
}
|
||||
|
||||
std::size_t Lease::live_count() { return live_.load(); }
|
||||
|
||||
bool Lease::acquire(std::int64_t ttl_ms) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (live_.load() > ::molo::operability::Caps::kMaxLeases) return false;
|
||||
if (key_.empty() || owner_.empty()) return false;
|
||||
if (ttl_ms <= 0) return false;
|
||||
if (ttl_ms > ::molo::operability::Caps::kMaxLeaseTtlMs) {
|
||||
ttl_ms = ::molo::operability::Caps::kMaxLeaseTtlMs;
|
||||
}
|
||||
if (!kv_.acquire_ms(key_, owner_, ttl_ms)) return false;
|
||||
holds_ = true;
|
||||
acquired_ms_ = clock_.steady_ms();
|
||||
observer_.event("lease.acquired", key_);
|
||||
observer_.counter("lease_acquired_total", 1);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Lease::renew(std::int64_t ttl_ms) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (!holds_) return false;
|
||||
if (ttl_ms <= 0) return false;
|
||||
if (ttl_ms > ::molo::operability::Caps::kMaxLeaseTtlMs) {
|
||||
ttl_ms = ::molo::operability::Caps::kMaxLeaseTtlMs;
|
||||
}
|
||||
if (!kv_.renew_ms(key_, owner_, ttl_ms)) {
|
||||
holds_ = false;
|
||||
observer_.event("lease.lost", key_);
|
||||
return false;
|
||||
}
|
||||
observer_.counter("lease_renewed_total", 1);
|
||||
return true;
|
||||
}
|
||||
|
||||
void Lease::release() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (!holds_) return;
|
||||
holds_ = false;
|
||||
std::string cur;
|
||||
if (kv_.get(key_, cur) && cur == owner_) kv_.erase(key_);
|
||||
observer_.event("lease.released", key_);
|
||||
}
|
||||
|
||||
bool Lease::holds() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return holds_;
|
||||
}
|
||||
|
||||
const std::string& Lease::key() const { return key_; }
|
||||
const std::string& Lease::owner() const { return owner_; }
|
||||
|
||||
} // namespace molo::coord
|
||||
@@ -0,0 +1,50 @@
|
||||
// Single distributed lease backed by the KV store. At most kMaxLeases live
|
||||
// Lease objects may exist per process; acquire maps to KvStore::acquire and
|
||||
// renew maps to KvStore::renew with the owner id as the expected value.
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
namespace molo::common {
|
||||
class Clock;
|
||||
class KvStore;
|
||||
class Observer;
|
||||
} // namespace molo::common
|
||||
|
||||
namespace molo::coord {
|
||||
|
||||
class Lease {
|
||||
public:
|
||||
Lease(::molo::common::KvStore& kv, ::molo::common::Clock& clock,
|
||||
::molo::common::Observer& observer, std::string key, std::string owner);
|
||||
~Lease();
|
||||
|
||||
Lease(const Lease&) = delete;
|
||||
Lease& operator=(const Lease&) = delete;
|
||||
|
||||
static std::size_t live_count();
|
||||
|
||||
bool acquire(std::int64_t ttl_ms);
|
||||
bool renew(std::int64_t ttl_ms);
|
||||
void release();
|
||||
bool holds() const;
|
||||
const std::string& key() const;
|
||||
const std::string& owner() const;
|
||||
|
||||
private:
|
||||
::molo::common::KvStore& kv_;
|
||||
::molo::common::Clock& clock_;
|
||||
::molo::common::Observer& observer_;
|
||||
std::string key_;
|
||||
std::string owner_;
|
||||
mutable std::mutex mutex_;
|
||||
bool holds_ = false;
|
||||
std::int64_t acquired_ms_ = 0;
|
||||
|
||||
static std::atomic<std::size_t> live_;
|
||||
};
|
||||
|
||||
} // namespace molo::coord
|
||||
@@ -0,0 +1,63 @@
|
||||
#include "operability/debug/endpoint.h"
|
||||
|
||||
#include "operability/caps.h"
|
||||
|
||||
namespace molo::debug {
|
||||
|
||||
Endpoint::Endpoint(SnapshotFn snapshot, AuthCheck auth)
|
||||
: snapshot_(std::move(snapshot)), auth_(std::move(auth)) {}
|
||||
|
||||
bool Endpoint::enabled() {
|
||||
#ifdef MOLO_ENABLE_DEBUG
|
||||
return true;
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
Endpoint::Response Endpoint::handle(const std::string& path, const std::string& token) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
Response r;
|
||||
if (!enabled()) {
|
||||
r.status = 404;
|
||||
r.body = "not found";
|
||||
return r;
|
||||
}
|
||||
if (path.size() > ::molo::operability::Caps::kDebugMaxPathBytes) {
|
||||
r.status = 414;
|
||||
r.body = "path too long";
|
||||
return r;
|
||||
}
|
||||
if (auth_) {
|
||||
bool ok = false;
|
||||
try {
|
||||
ok = auth_(token);
|
||||
} catch (...) {
|
||||
ok = false;
|
||||
}
|
||||
if (!ok) {
|
||||
r.status = 403;
|
||||
r.body = "forbidden";
|
||||
return r;
|
||||
}
|
||||
}
|
||||
if (path == "/debug/snapshot") {
|
||||
r.status = 200;
|
||||
r.content_type = "application/json";
|
||||
r.body = snapshot_ ? snapshot_() : "{}";
|
||||
} else if (path == "/debug/ping") {
|
||||
r.status = 200;
|
||||
r.content_type = "text/plain";
|
||||
r.body = "pong";
|
||||
} else {
|
||||
r.status = 404;
|
||||
r.body = "not found";
|
||||
return r;
|
||||
}
|
||||
if (r.body.size() > ::molo::operability::Caps::kDebugMaxResponseBytes) {
|
||||
r.body.resize(::molo::operability::Caps::kDebugMaxResponseBytes);
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
} // namespace molo::debug
|
||||
@@ -0,0 +1,39 @@
|
||||
// Admin only debug endpoints served under /debug/*. Production mounts this
|
||||
// on the admin listener only (public listener returns 404), behind the
|
||||
// MOLO_ENABLE_DEBUG compile flag plus the injected auth check. Responses
|
||||
// are capped at 1 MB.
|
||||
#pragma once
|
||||
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
namespace molo::debug {
|
||||
|
||||
class Endpoint {
|
||||
public:
|
||||
using SnapshotFn = std::function<std::string()>;
|
||||
using AuthCheck = std::function<bool(const std::string& token)>;
|
||||
|
||||
struct Response {
|
||||
int status = 404;
|
||||
std::string content_type = "text/plain";
|
||||
std::string body;
|
||||
};
|
||||
|
||||
Endpoint(SnapshotFn snapshot = SnapshotFn{}, AuthCheck auth = AuthCheck{});
|
||||
|
||||
Endpoint(const Endpoint&) = delete;
|
||||
Endpoint& operator=(const Endpoint&) = delete;
|
||||
|
||||
static bool enabled();
|
||||
|
||||
Response handle(const std::string& path, const std::string& token);
|
||||
|
||||
private:
|
||||
SnapshotFn snapshot_;
|
||||
AuthCheck auth_;
|
||||
mutable std::mutex mutex_;
|
||||
};
|
||||
|
||||
} // namespace molo::debug
|
||||
@@ -0,0 +1,89 @@
|
||||
#include "operability/debug/snapshot.h"
|
||||
|
||||
#include "operability/caps.h"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
|
||||
namespace molo::debug {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr std::size_t kMaxSections = 64;
|
||||
|
||||
std::string escape_json(const std::string& s) {
|
||||
std::string out;
|
||||
out.reserve(s.size());
|
||||
for (char c : s) {
|
||||
switch (c) {
|
||||
case '"':
|
||||
out += "\\\"";
|
||||
break;
|
||||
case '\\':
|
||||
out += "\\\\";
|
||||
break;
|
||||
case '\n':
|
||||
out += "\\n";
|
||||
break;
|
||||
case '\r':
|
||||
out += "\\r";
|
||||
break;
|
||||
case '\t':
|
||||
out += "\\t";
|
||||
break;
|
||||
default:
|
||||
out.push_back(c);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Snapshot::Snapshot(::molo::common::Clock& clock, std::string version)
|
||||
: clock_(clock), version_(std::move(version)), boot_ms_(clock.wall_ms()) {}
|
||||
|
||||
bool Snapshot::add_section(const std::string& name, const std::string& json) {
|
||||
if (name.empty()) return false;
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
for (const auto& s : sections_) {
|
||||
if (s.first == name) return false;
|
||||
}
|
||||
if (sections_.size() >= kMaxSections) return false;
|
||||
sections_.emplace_back(name, json);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Snapshot::remove_section(const std::string& name) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
for (auto it = sections_.begin(); it != sections_.end(); ++it) {
|
||||
if (it->first == name) {
|
||||
sections_.erase(it);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string Snapshot::capture() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
const std::int64_t now = clock_.wall_ms();
|
||||
std::string out = "{";
|
||||
out += "\"version\":\"" + escape_json(version_) + "\"";
|
||||
out += ",\"wall_ms\":" + std::to_string(now);
|
||||
out += ",\"uptime_ms\":" + std::to_string(now - boot_ms_);
|
||||
for (const auto& s : sections_) {
|
||||
out += ",\"" + escape_json(s.first) + "\":" + (s.second.empty() ? "null" : s.second);
|
||||
}
|
||||
out += "}";
|
||||
if (out.size() > ::molo::operability::Caps::kDebugMaxResponseBytes) {
|
||||
out.resize(::molo::operability::Caps::kDebugMaxResponseBytes);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::size_t Snapshot::section_count() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return sections_.size();
|
||||
}
|
||||
|
||||
} // namespace molo::debug
|
||||
@@ -0,0 +1,40 @@
|
||||
// Point in time debug snapshot: version, clocks and registered sections
|
||||
// rendered as one JSON document capped at 1 MB. Sections let each subsystem
|
||||
// contribute without the snapshot knowing their types.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace molo::common {
|
||||
class Clock;
|
||||
} // namespace molo::common
|
||||
|
||||
namespace molo::debug {
|
||||
|
||||
class Snapshot {
|
||||
public:
|
||||
explicit Snapshot(::molo::common::Clock& clock, std::string version = "0.0.0");
|
||||
|
||||
Snapshot(const Snapshot&) = delete;
|
||||
Snapshot& operator=(const Snapshot&) = delete;
|
||||
|
||||
// Register a pre-rendered JSON fragment under name. Returns false when the
|
||||
// section table is full or the name already exists.
|
||||
bool add_section(const std::string& name, const std::string& json);
|
||||
bool remove_section(const std::string& name);
|
||||
|
||||
std::string capture();
|
||||
std::size_t section_count() const;
|
||||
|
||||
private:
|
||||
::molo::common::Clock& clock_;
|
||||
std::string version_;
|
||||
std::int64_t boot_ms_;
|
||||
mutable std::mutex mutex_;
|
||||
std::vector<std::pair<std::string, std::string>> sections_;
|
||||
};
|
||||
|
||||
} // namespace molo::debug
|
||||
@@ -0,0 +1,205 @@
|
||||
#include "operability/deploy/blue_green.h"
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "operability/caps.h"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace molo::deploy {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr std::int64_t kPollMs = 50;
|
||||
|
||||
} // namespace
|
||||
|
||||
BlueGreen::BlueGreen(Config cfg, ::molo::common::Clock& clock, ::molo::common::Observer& observer,
|
||||
ReadyProbe probe, SwitchHook on_switch, RetireHook on_retire)
|
||||
: cfg_(cfg),
|
||||
clock_(clock),
|
||||
observer_(observer),
|
||||
probe_(std::move(probe)),
|
||||
on_switch_(std::move(on_switch)),
|
||||
on_retire_(std::move(on_retire)) {
|
||||
if (cfg_.green_instances > ::molo::operability::Caps::kMaxDeployInstances) {
|
||||
cfg_.green_instances = ::molo::operability::Caps::kMaxDeployInstances;
|
||||
}
|
||||
if (cfg_.ready_timeout_ms > ::molo::operability::Caps::kDeployReadyTimeoutMs) {
|
||||
cfg_.ready_timeout_ms = ::molo::operability::Caps::kDeployReadyTimeoutMs;
|
||||
}
|
||||
if (cfg_.ready_timeout_ms < 0) cfg_.ready_timeout_ms = 0;
|
||||
if (cfg_.rollback_window_ms > ::molo::operability::Caps::kBlueGreenRollbackWindowMs) {
|
||||
cfg_.rollback_window_ms = ::molo::operability::Caps::kBlueGreenRollbackWindowMs;
|
||||
}
|
||||
if (cfg_.rollback_window_ms < 0) cfg_.rollback_window_ms = 0;
|
||||
}
|
||||
|
||||
bool BlueGreen::start() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (state_ != State::kIdle) return false;
|
||||
if (cfg_.green_instances == 0 || !probe_) return false;
|
||||
state_ = State::kRunning;
|
||||
stage_ = Stage::kDeployGreen;
|
||||
serving_green_ = false;
|
||||
observer_.event("deploy.start", "blue_green green=" + std::to_string(cfg_.green_instances));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool BlueGreen::step() {
|
||||
Stage stage;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (state_ != State::kRunning) return state_ == State::kSucceeded;
|
||||
stage = stage_;
|
||||
}
|
||||
switch (stage) {
|
||||
case Stage::kDeployGreen: {
|
||||
observer_.event("deploy.stage", "deploy_green");
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
stage_ = Stage::kVerifyGreen;
|
||||
return true;
|
||||
}
|
||||
case Stage::kVerifyGreen: {
|
||||
if (!wait_green_ready()) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
state_ = State::kFailed;
|
||||
observer_.event("deploy.failed", "green never ready");
|
||||
return false;
|
||||
}
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
stage_ = Stage::kSwitch;
|
||||
return true;
|
||||
}
|
||||
case Stage::kSwitch: {
|
||||
bool ok = true;
|
||||
if (on_switch_) {
|
||||
try {
|
||||
ok = on_switch_();
|
||||
} catch (...) {
|
||||
ok = false;
|
||||
}
|
||||
}
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (!ok) {
|
||||
state_ = State::kFailed;
|
||||
observer_.event("deploy.failed", "switch refused");
|
||||
return false;
|
||||
}
|
||||
serving_green_ = true;
|
||||
window_end_ms_ = clock_.steady_ms() + cfg_.rollback_window_ms;
|
||||
stage_ = Stage::kRollbackWindow;
|
||||
observer_.event("deploy.stage", "serving green");
|
||||
return true;
|
||||
}
|
||||
case Stage::kRollbackWindow: {
|
||||
if (clock_.steady_ms() < window_end_ms_) return true; // stay here until window ends
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
stage_ = Stage::kRetireBlue;
|
||||
return true;
|
||||
}
|
||||
case Stage::kRetireBlue: {
|
||||
if (on_retire_) {
|
||||
try {
|
||||
on_retire_(true);
|
||||
} catch (...) {
|
||||
}
|
||||
}
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
stage_ = Stage::kDone;
|
||||
state_ = State::kSucceeded;
|
||||
observer_.event("deploy.succeeded", "blue retired");
|
||||
return true;
|
||||
}
|
||||
case Stage::kDone:
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void BlueGreen::abort() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (state_ == State::kRunning || state_ == State::kPaused) {
|
||||
state_ = State::kAborted;
|
||||
observer_.event("deploy.aborted", "serving_green=" + std::to_string(serving_green_));
|
||||
}
|
||||
}
|
||||
|
||||
BlueGreen::State BlueGreen::state() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return state_;
|
||||
}
|
||||
|
||||
const char* BlueGreen::state_name() const { return name_of(state()); }
|
||||
|
||||
bool BlueGreen::rollback() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (!serving_green_) return false;
|
||||
if (state_ != State::kRunning && state_ != State::kFailed) return false;
|
||||
if (state_ == State::kRunning && stage_ != Stage::kRollbackWindow) return false;
|
||||
serving_green_ = false;
|
||||
stage_ = Stage::kDone;
|
||||
state_ = State::kRolledBack;
|
||||
if (on_retire_) {
|
||||
try {
|
||||
on_retire_(false);
|
||||
} catch (...) {
|
||||
}
|
||||
}
|
||||
observer_.event("deploy.rollback", "back to blue");
|
||||
return true;
|
||||
}
|
||||
|
||||
BlueGreen::Stage BlueGreen::stage() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return stage_;
|
||||
}
|
||||
|
||||
bool BlueGreen::serving_green() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return serving_green_;
|
||||
}
|
||||
|
||||
const char* BlueGreen::name_of(State s) {
|
||||
switch (s) {
|
||||
case State::kIdle:
|
||||
return "idle";
|
||||
case State::kRunning:
|
||||
return "running";
|
||||
case State::kPaused:
|
||||
return "paused";
|
||||
case State::kSucceeded:
|
||||
return "succeeded";
|
||||
case State::kFailed:
|
||||
return "failed";
|
||||
case State::kAborted:
|
||||
return "aborted";
|
||||
case State::kRolledBack:
|
||||
return "rolled_back";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
bool BlueGreen::wait_green_ready() {
|
||||
const std::int64_t deadline = clock_.steady_ms() + cfg_.ready_timeout_ms;
|
||||
for (;;) {
|
||||
bool all_ready = true;
|
||||
for (std::size_t i = 0; i < cfg_.green_instances; ++i) {
|
||||
bool ready = false;
|
||||
try {
|
||||
ready = probe_(i);
|
||||
} catch (...) {
|
||||
ready = false;
|
||||
}
|
||||
if (!ready) {
|
||||
all_ready = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (all_ready) return true;
|
||||
if (clock_.steady_ms() >= deadline) return false;
|
||||
clock_.sleep_ms(kPollMs);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace molo::deploy
|
||||
@@ -0,0 +1,74 @@
|
||||
// Blue/green deployment: build the green fleet next to blue, verify it,
|
||||
// switch traffic, then retire blue after a rollback window. Every stage is
|
||||
// explicit so an operator or a frozen clock can drive it step by step.
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
|
||||
#include "operability/deploy/strategy.h"
|
||||
|
||||
namespace molo::common {
|
||||
class Clock;
|
||||
class Observer;
|
||||
} // namespace molo::common
|
||||
|
||||
namespace molo::deploy {
|
||||
|
||||
class BlueGreen : public Strategy {
|
||||
public:
|
||||
enum class Stage : int {
|
||||
kDeployGreen = 0,
|
||||
kVerifyGreen = 1,
|
||||
kSwitch = 2,
|
||||
kRollbackWindow = 3,
|
||||
kRetireBlue = 4,
|
||||
kDone = 5
|
||||
};
|
||||
|
||||
struct Config {
|
||||
std::size_t green_instances = 0;
|
||||
std::int64_t ready_timeout_ms = 60000;
|
||||
std::int64_t rollback_window_ms = 600000;
|
||||
};
|
||||
|
||||
using ReadyProbe = std::function<bool(std::size_t index)>;
|
||||
using SwitchHook = std::function<bool()>;
|
||||
using RetireHook = std::function<void(bool green)>;
|
||||
|
||||
BlueGreen(Config cfg, ::molo::common::Clock& clock, ::molo::common::Observer& observer,
|
||||
ReadyProbe probe, SwitchHook on_switch = SwitchHook{},
|
||||
RetireHook on_retire = RetireHook{});
|
||||
|
||||
bool start() override;
|
||||
bool step() override;
|
||||
void abort() override;
|
||||
State state() const override;
|
||||
const char* state_name() const override;
|
||||
|
||||
// Roll back to blue. Valid during the rollback window and after failure.
|
||||
bool rollback();
|
||||
|
||||
Stage stage() const;
|
||||
bool serving_green() const;
|
||||
|
||||
private:
|
||||
static const char* name_of(State s);
|
||||
bool wait_green_ready();
|
||||
|
||||
Config cfg_;
|
||||
::molo::common::Clock& clock_;
|
||||
::molo::common::Observer& observer_;
|
||||
ReadyProbe probe_;
|
||||
SwitchHook on_switch_;
|
||||
RetireHook on_retire_;
|
||||
mutable std::mutex mutex_;
|
||||
State state_ = State::kIdle;
|
||||
Stage stage_ = Stage::kDeployGreen;
|
||||
bool serving_green_ = false;
|
||||
std::int64_t window_end_ms_ = 0;
|
||||
};
|
||||
|
||||
} // namespace molo::deploy
|
||||
@@ -0,0 +1,137 @@
|
||||
#include "operability/deploy/lifecycle.h"
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "operability/caps.h"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace molo::deploy {
|
||||
|
||||
Lifecycle::Lifecycle(::molo::common::Clock& clock, ::molo::common::Observer& observer)
|
||||
: clock_(clock), observer_(observer) {}
|
||||
|
||||
void Lifecycle::on_stop_listeners(Hook hook) { stop_listeners_ = std::move(hook); }
|
||||
void Lifecycle::on_connections_drained(DrainedProbe probe) {
|
||||
connections_drained_ = std::move(probe);
|
||||
}
|
||||
void Lifecycle::on_background_drained(DrainedProbe probe) {
|
||||
background_drained_ = std::move(probe);
|
||||
}
|
||||
void Lifecycle::on_drop_background(DropHook hook) { drop_background_ = std::move(hook); }
|
||||
void Lifecycle::on_serving(Hook hook) { on_serving_ = std::move(hook); }
|
||||
void Lifecycle::on_complete(Hook hook) { on_complete_ = std::move(hook); }
|
||||
|
||||
void Lifecycle::mark_serving() {
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
phase_ = Phase::kServing;
|
||||
}
|
||||
observer_.event("lifecycle.serving", "");
|
||||
if (on_serving_) {
|
||||
try {
|
||||
on_serving_();
|
||||
} catch (...) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool Lifecycle::run_drain(std::int64_t deadline_ms, std::int64_t poll_ms) {
|
||||
if (deadline_ms > ::molo::operability::Caps::kDrainDeadlineMs) {
|
||||
deadline_ms = ::molo::operability::Caps::kDrainDeadlineMs;
|
||||
}
|
||||
if (deadline_ms < 0) deadline_ms = 0;
|
||||
if (poll_ms <= 0) poll_ms = ::molo::operability::Caps::kDrainPollMs;
|
||||
|
||||
const std::int64_t start = clock_.steady_ms();
|
||||
const std::int64_t deadline = start + deadline_ms;
|
||||
bool dropped = false;
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
phase_ = Phase::kStopListeners;
|
||||
}
|
||||
observer_.event("lifecycle.phase", "stop_listeners");
|
||||
if (stop_listeners_) {
|
||||
try {
|
||||
stop_listeners_();
|
||||
} catch (...) {
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
phase_ = Phase::kDrainConnections;
|
||||
}
|
||||
const bool conn_ok = drain_until(connections_drained_, deadline, poll_ms, "connections");
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
phase_ = Phase::kDrainBackground;
|
||||
}
|
||||
const bool bg_ok = drain_until(background_drained_, deadline, poll_ms, "background");
|
||||
if (!bg_ok) {
|
||||
dropped = true;
|
||||
observer_.event("lifecycle.dropped", "background work exceeded drain deadline");
|
||||
if (drop_background_) {
|
||||
try {
|
||||
drop_background_("drain deadline exceeded");
|
||||
} catch (...) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
phase_ = Phase::kFinalize;
|
||||
}
|
||||
observer_.event("lifecycle.phase", "finalize");
|
||||
if (on_complete_) {
|
||||
try {
|
||||
on_complete_();
|
||||
} catch (...) {
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
phase_ = Phase::kDone;
|
||||
clean_ = conn_ok && bg_ok && !dropped;
|
||||
}
|
||||
observer_.event("lifecycle.done", clean_ ? "clean" : "forced");
|
||||
return clean_;
|
||||
}
|
||||
|
||||
Lifecycle::Phase Lifecycle::phase() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return phase_;
|
||||
}
|
||||
|
||||
bool Lifecycle::clean() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return clean_;
|
||||
}
|
||||
|
||||
bool Lifecycle::drain_until(const DrainedProbe& probe, std::int64_t deadline_ms,
|
||||
std::int64_t poll_ms, const char* what) {
|
||||
if (!probe) return true;
|
||||
for (;;) {
|
||||
bool drained = false;
|
||||
try {
|
||||
drained = probe();
|
||||
} catch (...) {
|
||||
drained = false;
|
||||
}
|
||||
if (drained) {
|
||||
observer_.event("lifecycle.drained", what);
|
||||
return true;
|
||||
}
|
||||
if (clock_.steady_ms() >= deadline_ms) {
|
||||
observer_.event("lifecycle.timeout", what);
|
||||
return false;
|
||||
}
|
||||
clock_.sleep_ms(poll_ms);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace molo::deploy
|
||||
@@ -0,0 +1,71 @@
|
||||
// Process lifecycle: boot completion plus a four phase graceful drain.
|
||||
// Phase 1 stops listeners, phase 2 drains connections, phase 3 drains
|
||||
// background work (timed out work is handed to the drop hook, which wires
|
||||
// to the dead letter queue), phase 4 finalizes. Production wires the hooks
|
||||
// to Listener::stop, Connection::drain, ShutdownCoordinator and StartupGate.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
namespace molo::common {
|
||||
class Clock;
|
||||
class Observer;
|
||||
} // namespace molo::common
|
||||
|
||||
namespace molo::deploy {
|
||||
|
||||
class Lifecycle {
|
||||
public:
|
||||
using Hook = std::function<void()>;
|
||||
using DrainedProbe = std::function<bool()>;
|
||||
using DropHook = std::function<void(const std::string& reason)>;
|
||||
|
||||
enum class Phase : int {
|
||||
kBoot = 0,
|
||||
kServing = 1,
|
||||
kStopListeners = 2,
|
||||
kDrainConnections = 3,
|
||||
kDrainBackground = 4,
|
||||
kFinalize = 5,
|
||||
kDone = 6
|
||||
};
|
||||
|
||||
Lifecycle(::molo::common::Clock& clock, ::molo::common::Observer& observer);
|
||||
|
||||
void on_stop_listeners(Hook hook);
|
||||
void on_connections_drained(DrainedProbe probe);
|
||||
void on_background_drained(DrainedProbe probe);
|
||||
void on_drop_background(DropHook hook);
|
||||
void on_serving(Hook hook);
|
||||
void on_complete(Hook hook);
|
||||
|
||||
// Called once the server accepts traffic. Wires to StartupGate.
|
||||
void mark_serving();
|
||||
|
||||
// Runs all four drain phases. Returns true when fully drained.
|
||||
bool run_drain(std::int64_t deadline_ms, std::int64_t poll_ms = 50);
|
||||
|
||||
Phase phase() const;
|
||||
bool clean() const;
|
||||
|
||||
private:
|
||||
bool drain_until(const DrainedProbe& probe, std::int64_t deadline_ms, std::int64_t poll_ms,
|
||||
const char* what);
|
||||
|
||||
::molo::common::Clock& clock_;
|
||||
::molo::common::Observer& observer_;
|
||||
Hook stop_listeners_;
|
||||
DrainedProbe connections_drained_;
|
||||
DrainedProbe background_drained_;
|
||||
DropHook drop_background_;
|
||||
Hook on_serving_;
|
||||
Hook on_complete_;
|
||||
mutable std::mutex mutex_;
|
||||
Phase phase_ = Phase::kBoot;
|
||||
bool clean_ = false;
|
||||
};
|
||||
|
||||
} // namespace molo::deploy
|
||||
@@ -0,0 +1,152 @@
|
||||
#include "operability/deploy/rolling.h"
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "operability/caps.h"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace molo::deploy {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr std::int64_t kPollMs = 50;
|
||||
|
||||
} // namespace
|
||||
|
||||
Rolling::Rolling(Config cfg, ::molo::common::Clock& clock, ::molo::common::Observer& observer,
|
||||
ReadyProbe probe, UpgradeHook hook)
|
||||
: cfg_(cfg), clock_(clock), observer_(observer), probe_(std::move(probe)), hook_(std::move(hook)) {
|
||||
if (cfg_.total_instances > ::molo::operability::Caps::kMaxDeployInstances) {
|
||||
cfg_.total_instances = ::molo::operability::Caps::kMaxDeployInstances;
|
||||
}
|
||||
if (cfg_.batch_size == 0) cfg_.batch_size = 1;
|
||||
if (cfg_.batch_size > ::molo::operability::Caps::kMaxDeployBatch) {
|
||||
cfg_.batch_size = ::molo::operability::Caps::kMaxDeployBatch;
|
||||
}
|
||||
if (cfg_.ready_timeout_ms > ::molo::operability::Caps::kDeployReadyTimeoutMs) {
|
||||
cfg_.ready_timeout_ms = ::molo::operability::Caps::kDeployReadyTimeoutMs;
|
||||
}
|
||||
if (cfg_.ready_timeout_ms < 0) cfg_.ready_timeout_ms = 0;
|
||||
if (cfg_.pause_between_batches_ms < 0) cfg_.pause_between_batches_ms = 0;
|
||||
}
|
||||
|
||||
bool Rolling::start() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (state_ != State::kIdle) return false;
|
||||
if (cfg_.total_instances == 0 || !probe_) return false;
|
||||
state_ = State::kRunning;
|
||||
cursor_ = 0;
|
||||
upgraded_ = 0;
|
||||
failed_ = 0;
|
||||
observer_.event("deploy.start", "rolling total=" + std::to_string(cfg_.total_instances));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Rolling::step() {
|
||||
std::size_t begin = 0;
|
||||
std::size_t end = 0;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (state_ != State::kRunning) return state_ == State::kSucceeded;
|
||||
begin = cursor_;
|
||||
end = begin + cfg_.batch_size;
|
||||
if (end > cfg_.total_instances) end = cfg_.total_instances;
|
||||
}
|
||||
for (std::size_t i = begin; i < end; ++i) {
|
||||
if (hook_) hook_(i);
|
||||
if (wait_ready(i)) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
++upgraded_;
|
||||
observer_.counter("deploy_upgraded_total", 1);
|
||||
} else {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
++failed_;
|
||||
observer_.counter("deploy_failed_total", 1);
|
||||
observer_.event("deploy.failed", "index=" + std::to_string(i));
|
||||
if (failed_ > cfg_.max_unavailable) {
|
||||
state_ = State::kFailed;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
cursor_ = end;
|
||||
observer_.event("deploy.batch", "cursor=" + std::to_string(cursor_));
|
||||
if (cursor_ >= cfg_.total_instances) {
|
||||
state_ = failed_ == 0 ? State::kSucceeded : State::kFailed;
|
||||
if (state_ == State::kSucceeded) observer_.event("deploy.succeeded", "");
|
||||
return state_ == State::kSucceeded;
|
||||
}
|
||||
}
|
||||
if (cfg_.pause_between_batches_ms > 0) clock_.sleep_ms(cfg_.pause_between_batches_ms);
|
||||
return true;
|
||||
}
|
||||
|
||||
void Rolling::abort() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (state_ == State::kRunning || state_ == State::kPaused) {
|
||||
state_ = State::kAborted;
|
||||
observer_.event("deploy.aborted", "cursor=" + std::to_string(cursor_));
|
||||
}
|
||||
}
|
||||
|
||||
Rolling::State Rolling::state() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return state_;
|
||||
}
|
||||
|
||||
const char* Rolling::state_name() const { return name_of(state()); }
|
||||
|
||||
std::size_t Rolling::upgraded() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return upgraded_;
|
||||
}
|
||||
|
||||
std::size_t Rolling::failed() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return failed_;
|
||||
}
|
||||
|
||||
std::size_t Rolling::pending() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return cfg_.total_instances > cursor_ ? cfg_.total_instances - cursor_ : 0;
|
||||
}
|
||||
|
||||
const char* Rolling::name_of(State s) {
|
||||
switch (s) {
|
||||
case State::kIdle:
|
||||
return "idle";
|
||||
case State::kRunning:
|
||||
return "running";
|
||||
case State::kPaused:
|
||||
return "paused";
|
||||
case State::kSucceeded:
|
||||
return "succeeded";
|
||||
case State::kFailed:
|
||||
return "failed";
|
||||
case State::kAborted:
|
||||
return "aborted";
|
||||
case State::kRolledBack:
|
||||
return "rolled_back";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
bool Rolling::wait_ready(std::size_t index) {
|
||||
const std::int64_t deadline = clock_.steady_ms() + cfg_.ready_timeout_ms;
|
||||
for (;;) {
|
||||
bool ready = false;
|
||||
try {
|
||||
ready = probe_(index);
|
||||
} catch (...) {
|
||||
ready = false;
|
||||
}
|
||||
if (ready) return true;
|
||||
if (clock_.steady_ms() >= deadline) return false;
|
||||
clock_.sleep_ms(kPollMs);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace molo::deploy
|
||||
@@ -0,0 +1,62 @@
|
||||
// Rolling deployment: upgrade instances in batches, waiting for each batch
|
||||
// to report ready before moving on. Readiness comes from an injected probe
|
||||
// so production can poll readyz/startupz while tests use a stub.
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
|
||||
#include "operability/deploy/strategy.h"
|
||||
|
||||
namespace molo::common {
|
||||
class Clock;
|
||||
class Observer;
|
||||
} // namespace molo::common
|
||||
|
||||
namespace molo::deploy {
|
||||
|
||||
class Rolling : public Strategy {
|
||||
public:
|
||||
struct Config {
|
||||
std::size_t total_instances = 0;
|
||||
std::size_t batch_size = 1;
|
||||
std::int64_t pause_between_batches_ms = 0;
|
||||
std::int64_t ready_timeout_ms = 60000;
|
||||
std::size_t max_unavailable = 0;
|
||||
};
|
||||
|
||||
using ReadyProbe = std::function<bool(std::size_t index)>;
|
||||
using UpgradeHook = std::function<void(std::size_t index)>;
|
||||
|
||||
Rolling(Config cfg, ::molo::common::Clock& clock, ::molo::common::Observer& observer,
|
||||
ReadyProbe probe, UpgradeHook hook = UpgradeHook{});
|
||||
|
||||
bool start() override;
|
||||
bool step() override;
|
||||
void abort() override;
|
||||
State state() const override;
|
||||
const char* state_name() const override;
|
||||
|
||||
std::size_t upgraded() const;
|
||||
std::size_t failed() const;
|
||||
std::size_t pending() const;
|
||||
|
||||
private:
|
||||
static const char* name_of(State s);
|
||||
bool wait_ready(std::size_t index);
|
||||
|
||||
Config cfg_;
|
||||
::molo::common::Clock& clock_;
|
||||
::molo::common::Observer& observer_;
|
||||
ReadyProbe probe_;
|
||||
UpgradeHook hook_;
|
||||
mutable std::mutex mutex_;
|
||||
State state_ = State::kIdle;
|
||||
std::size_t cursor_ = 0;
|
||||
std::size_t upgraded_ = 0;
|
||||
std::size_t failed_ = 0;
|
||||
};
|
||||
|
||||
} // namespace molo::deploy
|
||||
@@ -0,0 +1,29 @@
|
||||
// Common interface for deployment strategies (rolling, blue/green).
|
||||
// Strategies are driven step by step so the caller keeps the event loop
|
||||
// responsive; every blocking wait goes through the injected clock.
|
||||
#pragma once
|
||||
|
||||
namespace molo::deploy {
|
||||
|
||||
class Strategy {
|
||||
public:
|
||||
enum class State : int {
|
||||
kIdle = 0,
|
||||
kRunning = 1,
|
||||
kPaused = 2,
|
||||
kSucceeded = 3,
|
||||
kFailed = 4,
|
||||
kAborted = 5,
|
||||
kRolledBack = 6
|
||||
};
|
||||
|
||||
virtual ~Strategy() = default;
|
||||
|
||||
virtual bool start() = 0;
|
||||
virtual bool step() = 0;
|
||||
virtual void abort() = 0;
|
||||
virtual State state() const = 0;
|
||||
virtual const char* state_name() const = 0;
|
||||
};
|
||||
|
||||
} // namespace molo::deploy
|
||||
@@ -0,0 +1,114 @@
|
||||
#include "operability/resilience/breaker.h"
|
||||
|
||||
#include "operability/caps.h"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace molo::resilience {
|
||||
|
||||
Breaker::Breaker(::molo::common::Clock& clock, ::molo::common::Observer& observer, std::string name)
|
||||
: clock_(clock), observer_(observer), name_(std::move(name)) {}
|
||||
|
||||
bool Breaker::allow() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
const std::int64_t now = clock_.steady_ms();
|
||||
if (state_ == State::kClosed) return true;
|
||||
if (state_ == State::kOpen) {
|
||||
if (now - opened_ms_ < ::molo::operability::Caps::kBreakerOpenHoldMs) return false;
|
||||
state_ = State::kHalfOpen;
|
||||
half_open_inflight_ = 0;
|
||||
half_open_ok_ = 0;
|
||||
observer_.event("breaker.half_open", name_);
|
||||
}
|
||||
if (half_open_inflight_ >= ::molo::operability::Caps::kBreakerHalfOpenProbes) return false;
|
||||
++half_open_inflight_;
|
||||
return true;
|
||||
}
|
||||
|
||||
void Breaker::on_success() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
const std::int64_t now = clock_.steady_ms();
|
||||
consecutive_failures_ = 0;
|
||||
if (state_ == State::kClosed) {
|
||||
samples_.push_back(now);
|
||||
prune_locked(now);
|
||||
return;
|
||||
}
|
||||
if (state_ == State::kHalfOpen) {
|
||||
if (half_open_inflight_ > 0) --half_open_inflight_;
|
||||
++half_open_ok_;
|
||||
if (half_open_ok_ >= ::molo::operability::Caps::kBreakerHalfOpenProbes) {
|
||||
state_ = State::kClosed;
|
||||
failures_.clear();
|
||||
samples_.clear();
|
||||
observer_.event("breaker.closed", name_);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Breaker::on_failure() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
const std::int64_t now = clock_.steady_ms();
|
||||
++consecutive_failures_;
|
||||
observer_.counter("breaker_failures_total", 1);
|
||||
if (state_ == State::kHalfOpen) {
|
||||
if (half_open_inflight_ > 0) --half_open_inflight_;
|
||||
trip_locked(now);
|
||||
return;
|
||||
}
|
||||
if (state_ == State::kOpen) return;
|
||||
failures_.push_back(now);
|
||||
samples_.push_back(now);
|
||||
prune_locked(now);
|
||||
if (consecutive_failures_ >=
|
||||
static_cast<std::size_t>(::molo::operability::Caps::kBreakerConsecutiveFailures)) {
|
||||
trip_locked(now);
|
||||
return;
|
||||
}
|
||||
if (samples_.size() >=
|
||||
static_cast<std::size_t>(::molo::operability::Caps::kBreakerWindowMinSamples) &&
|
||||
failures_.size() * 100 >=
|
||||
samples_.size() *
|
||||
static_cast<std::size_t>(::molo::operability::Caps::kBreakerWindowFailurePct)) {
|
||||
trip_locked(now);
|
||||
}
|
||||
}
|
||||
|
||||
Breaker::State Breaker::state() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return state_;
|
||||
}
|
||||
|
||||
const char* Breaker::state_name() const {
|
||||
switch (state()) {
|
||||
case State::kClosed:
|
||||
return "closed";
|
||||
case State::kOpen:
|
||||
return "open";
|
||||
case State::kHalfOpen:
|
||||
return "half_open";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
const std::string& Breaker::name() const { return name_; }
|
||||
|
||||
void Breaker::trip_locked(std::int64_t now) {
|
||||
state_ = State::kOpen;
|
||||
opened_ms_ = now;
|
||||
half_open_inflight_ = 0;
|
||||
half_open_ok_ = 0;
|
||||
observer_.event("breaker.open", name_);
|
||||
observer_.counter("breaker_opened_total", 1);
|
||||
}
|
||||
|
||||
void Breaker::prune_locked(std::int64_t now) {
|
||||
const std::int64_t cutoff = now - ::molo::operability::Caps::kBreakerWindowMs;
|
||||
while (!failures_.empty() && failures_.front() < cutoff) failures_.pop_front();
|
||||
while (!samples_.empty() && samples_.front() < cutoff) samples_.pop_front();
|
||||
constexpr std::size_t kCap = 4096;
|
||||
while (failures_.size() > kCap) failures_.pop_front();
|
||||
while (samples_.size() > kCap) samples_.pop_front();
|
||||
}
|
||||
|
||||
} // namespace molo::resilience
|
||||
@@ -0,0 +1,55 @@
|
||||
// Circuit breaker, innermost resilience stage: never retry into an open
|
||||
// breaker. Opens after 20 consecutive failures or a 50 percent failure rate
|
||||
// over a 10 second window (minimum 10 samples). Half open admits 3 probes;
|
||||
// all succeeding closes the breaker, any failing reopens it.
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <deque>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
namespace molo::common {
|
||||
class Clock;
|
||||
class Observer;
|
||||
} // namespace molo::common
|
||||
|
||||
namespace molo::resilience {
|
||||
|
||||
class Breaker {
|
||||
public:
|
||||
enum class State : int { kClosed = 0, kOpen = 1, kHalfOpen = 2 };
|
||||
|
||||
Breaker(::molo::common::Clock& clock, ::molo::common::Observer& observer, std::string name);
|
||||
|
||||
Breaker(const Breaker&) = delete;
|
||||
Breaker& operator=(const Breaker&) = delete;
|
||||
|
||||
// Consult before sending. False means fail fast without calling downstream.
|
||||
bool allow();
|
||||
void on_success();
|
||||
void on_failure();
|
||||
|
||||
State state() const;
|
||||
const char* state_name() const;
|
||||
const std::string& name() const;
|
||||
|
||||
private:
|
||||
void trip_locked(std::int64_t now);
|
||||
void prune_locked(std::int64_t now);
|
||||
|
||||
::molo::common::Clock& clock_;
|
||||
::molo::common::Observer& observer_;
|
||||
std::string name_;
|
||||
mutable std::mutex mutex_;
|
||||
State state_ = State::kClosed;
|
||||
std::size_t consecutive_failures_ = 0;
|
||||
std::deque<std::int64_t> failures_;
|
||||
std::deque<std::int64_t> samples_;
|
||||
std::int64_t opened_ms_ = 0;
|
||||
int half_open_inflight_ = 0;
|
||||
int half_open_ok_ = 0;
|
||||
};
|
||||
|
||||
} // namespace molo::resilience
|
||||
@@ -0,0 +1,75 @@
|
||||
#include "operability/resilience/bulkhead.h"
|
||||
|
||||
#include <functional>
|
||||
|
||||
#include "operability/caps.h"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace molo::resilience {
|
||||
|
||||
Bulkhead::Bulkhead(::molo::common::Observer& observer, std::size_t partitions,
|
||||
std::size_t max_concurrent, std::size_t max_queue)
|
||||
: observer_(observer),
|
||||
partitions_(partitions),
|
||||
max_concurrent_(max_concurrent),
|
||||
max_queue_(max_queue) {
|
||||
if (partitions_ == 0) partitions_ = 1;
|
||||
if (partitions_ > ::molo::operability::Caps::kBulkheadMaxPartitions) {
|
||||
partitions_ = ::molo::operability::Caps::kBulkheadMaxPartitions;
|
||||
}
|
||||
if (max_concurrent_ == 0) max_concurrent_ = 1;
|
||||
if (max_concurrent_ > ::molo::operability::Caps::kBulkheadMaxConcurrent) {
|
||||
max_concurrent_ = ::molo::operability::Caps::kBulkheadMaxConcurrent;
|
||||
}
|
||||
if (max_queue_ > ::molo::operability::Caps::kBulkheadMaxQueue) {
|
||||
max_queue_ = ::molo::operability::Caps::kBulkheadMaxQueue;
|
||||
}
|
||||
}
|
||||
|
||||
Bulkhead::Admission Bulkhead::try_acquire(const std::string& partition) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
Slot& slot = slots_[slot_of(partition)];
|
||||
if (slot.inflight < max_concurrent_) {
|
||||
++slot.inflight;
|
||||
observer_.counter("bulkhead_admitted_total", 1);
|
||||
return Admission::kAdmitted;
|
||||
}
|
||||
if (slot.queued < max_queue_) {
|
||||
++slot.queued;
|
||||
observer_.counter("bulkhead_queued_total", 1);
|
||||
return Admission::kQueued;
|
||||
}
|
||||
observer_.counter("bulkhead_rejected_total", 1);
|
||||
return Admission::kRejected;
|
||||
}
|
||||
|
||||
void Bulkhead::release(const std::string& partition) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
Slot& slot = slots_[slot_of(partition)];
|
||||
if (slot.queued > 0) {
|
||||
// A queued request takes the freed slot; inflight count is unchanged.
|
||||
--slot.queued;
|
||||
observer_.counter("bulkhead_promoted_total", 1);
|
||||
return;
|
||||
}
|
||||
if (slot.inflight > 0) --slot.inflight;
|
||||
}
|
||||
|
||||
std::size_t Bulkhead::inflight(const std::string& partition) const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return slots_[slot_of(partition)].inflight;
|
||||
}
|
||||
|
||||
std::size_t Bulkhead::queued(const std::string& partition) const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return slots_[slot_of(partition)].queued;
|
||||
}
|
||||
|
||||
std::size_t Bulkhead::partition_count() const { return partitions_; }
|
||||
|
||||
std::size_t Bulkhead::slot_of(const std::string& partition) const {
|
||||
static const std::hash<std::string> kHash;
|
||||
return kHash(partition) % partitions_;
|
||||
}
|
||||
|
||||
} // namespace molo::resilience
|
||||
@@ -0,0 +1,49 @@
|
||||
// Partitioned bulkhead: at most 32 partitions, each with 64 concurrent
|
||||
// slots and a 128 deep queue. Excess work is rejected immediately so the
|
||||
// caller can shed with 503 plus Retry-After instead of queueing forever.
|
||||
#pragma once
|
||||
|
||||
#include <array>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
namespace molo::common {
|
||||
class Observer;
|
||||
} // namespace molo::common
|
||||
|
||||
namespace molo::resilience {
|
||||
|
||||
class Bulkhead {
|
||||
public:
|
||||
enum class Admission : int { kAdmitted = 0, kQueued = 1, kRejected = 2 };
|
||||
|
||||
Bulkhead(::molo::common::Observer& observer, std::size_t partitions = 8,
|
||||
std::size_t max_concurrent = 64, std::size_t max_queue = 128);
|
||||
|
||||
Bulkhead(const Bulkhead&) = delete;
|
||||
Bulkhead& operator=(const Bulkhead&) = delete;
|
||||
|
||||
Admission try_acquire(const std::string& partition);
|
||||
void release(const std::string& partition);
|
||||
std::size_t inflight(const std::string& partition) const;
|
||||
std::size_t queued(const std::string& partition) const;
|
||||
std::size_t partition_count() const;
|
||||
|
||||
private:
|
||||
struct Slot {
|
||||
std::size_t inflight = 0;
|
||||
std::size_t queued = 0;
|
||||
};
|
||||
std::size_t slot_of(const std::string& partition) const;
|
||||
|
||||
::molo::common::Observer& observer_;
|
||||
std::size_t partitions_;
|
||||
std::size_t max_concurrent_;
|
||||
std::size_t max_queue_;
|
||||
mutable std::mutex mutex_;
|
||||
std::array<Slot, 32> slots_;
|
||||
};
|
||||
|
||||
} // namespace molo::resilience
|
||||
@@ -0,0 +1,36 @@
|
||||
#include "operability/resilience/deadline.h"
|
||||
|
||||
#include "operability/caps.h"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
|
||||
namespace molo::resilience {
|
||||
|
||||
Deadline::Deadline(::molo::common::Clock& clock) : clock_(clock) {}
|
||||
|
||||
std::optional<std::int64_t> Deadline::parse_header(const std::string& value) {
|
||||
if (value.empty() || value.size() > 20) return std::nullopt;
|
||||
std::int64_t v = 0;
|
||||
for (char c : value) {
|
||||
if (c < '0' || c > '9') return std::nullopt;
|
||||
const std::int64_t next = v * 10 + static_cast<std::int64_t>(c - '0');
|
||||
if (next < v) return std::nullopt; // overflow
|
||||
v = next;
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
std::string Deadline::format_header(std::int64_t absolute_ms) {
|
||||
if (absolute_ms < 0) absolute_ms = 0;
|
||||
return std::to_string(absolute_ms);
|
||||
}
|
||||
|
||||
std::int64_t Deadline::remaining_ms(std::int64_t absolute_ms) const {
|
||||
const std::int64_t now = clock_.wall_ms();
|
||||
const std::int64_t remaining = absolute_ms - now;
|
||||
const std::int64_t budget = remaining - ::molo::operability::Caps::kDeadlineFloorMarginMs;
|
||||
return budget > 0 ? budget : 0;
|
||||
}
|
||||
|
||||
bool Deadline::expired(std::int64_t absolute_ms) const { return remaining_ms(absolute_ms) == 0; }
|
||||
|
||||
} // namespace molo::resilience
|
||||
@@ -0,0 +1,41 @@
|
||||
// Deadline propagation for the X-Deadline-Ms header. Inbound absolute
|
||||
// deadlines are converted to a remaining budget minus a 50 ms floor margin
|
||||
// before forwarding, so downstream work never starts with a budget that is
|
||||
// already expired. Production wires this into clients before send_inner.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
namespace molo::common {
|
||||
class Clock;
|
||||
} // namespace molo::common
|
||||
|
||||
namespace molo::resilience {
|
||||
|
||||
class Deadline {
|
||||
public:
|
||||
explicit Deadline(::molo::common::Clock& clock);
|
||||
|
||||
Deadline(const Deadline&) = delete;
|
||||
Deadline& operator=(const Deadline&) = delete;
|
||||
|
||||
// Parse an absolute epoch millis header value. Empty on malformed input.
|
||||
static std::optional<std::int64_t> parse_header(const std::string& value);
|
||||
|
||||
// Format an absolute epoch millis header value.
|
||||
static std::string format_header(std::int64_t absolute_ms);
|
||||
|
||||
// Remaining budget for outbound use: absolute minus now minus the 50 ms
|
||||
// floor margin. Returns 0 when already expired or within the margin.
|
||||
std::int64_t remaining_ms(std::int64_t absolute_ms) const;
|
||||
|
||||
// True when the absolute deadline leaves no usable budget.
|
||||
bool expired(std::int64_t absolute_ms) const;
|
||||
|
||||
private:
|
||||
::molo::common::Clock& clock_;
|
||||
};
|
||||
|
||||
} // namespace molo::resilience
|
||||
@@ -0,0 +1,44 @@
|
||||
#include "operability/resilience/shedder.h"
|
||||
|
||||
#include "operability/caps.h"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace molo::resilience {
|
||||
|
||||
Shedder::Shedder(::molo::common::Observer& observer, OverloadProbe probe,
|
||||
std::int64_t retry_after_s)
|
||||
: observer_(observer), probe_(std::move(probe)), retry_after_s_(retry_after_s) {
|
||||
if (retry_after_s_ < 0) retry_after_s_ = 0;
|
||||
if (retry_after_s_ > ::molo::operability::Caps::kShedRetryAfterMaxS) {
|
||||
retry_after_s_ = ::molo::operability::Caps::kShedRetryAfterMaxS;
|
||||
}
|
||||
}
|
||||
|
||||
bool Shedder::should_shed() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (!probe_) return false;
|
||||
bool shed = false;
|
||||
try {
|
||||
shed = probe_();
|
||||
} catch (...) {
|
||||
shed = false;
|
||||
}
|
||||
if (shed) observer_.counter("shed_total", 1);
|
||||
return shed;
|
||||
}
|
||||
|
||||
Shedder::Rejection Shedder::reject() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
Rejection r;
|
||||
r.status = 503;
|
||||
r.retry_after = std::to_string(retry_after_s_);
|
||||
observer_.counter("shed_rejected_total", 1);
|
||||
return r;
|
||||
}
|
||||
|
||||
std::int64_t Shedder::retry_after_s() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return retry_after_s_;
|
||||
}
|
||||
|
||||
} // namespace molo::resilience
|
||||
@@ -0,0 +1,44 @@
|
||||
// Load shedder: when the overload probe reports pressure, requests are
|
||||
// rejected with 503 plus a Retry-After hint instead of queueing. The probe
|
||||
// is injected so production can use queue depth or latency while tests use
|
||||
// a flag. Sits outside the bulkhead: shed first, then bulkhead, then send.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
namespace molo::common {
|
||||
class Observer;
|
||||
} // namespace molo::common
|
||||
|
||||
namespace molo::resilience {
|
||||
|
||||
class Shedder {
|
||||
public:
|
||||
using OverloadProbe = std::function<bool()>;
|
||||
|
||||
struct Rejection {
|
||||
int status = 503;
|
||||
std::string retry_after; // seconds, ASCII digits
|
||||
};
|
||||
|
||||
Shedder(::molo::common::Observer& observer, OverloadProbe probe,
|
||||
std::int64_t retry_after_s = 5);
|
||||
|
||||
Shedder(const Shedder&) = delete;
|
||||
Shedder& operator=(const Shedder&) = delete;
|
||||
|
||||
bool should_shed() const;
|
||||
Rejection reject() const;
|
||||
std::int64_t retry_after_s() const;
|
||||
|
||||
private:
|
||||
::molo::common::Observer& observer_;
|
||||
OverloadProbe probe_;
|
||||
std::int64_t retry_after_s_;
|
||||
mutable std::mutex mutex_;
|
||||
};
|
||||
|
||||
} // namespace molo::resilience
|
||||
@@ -0,0 +1,182 @@
|
||||
#include "operability/resilience/slo.h"
|
||||
|
||||
#include <cstdio>
|
||||
|
||||
#include "operability/caps.h"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace molo::resilience {
|
||||
|
||||
namespace {
|
||||
|
||||
std::string trunc(const std::string& s) {
|
||||
if (s.size() > 128) return s.substr(0, 128);
|
||||
return s;
|
||||
}
|
||||
|
||||
std::string escape_label(const std::string& s) {
|
||||
std::string out;
|
||||
out.reserve(s.size());
|
||||
for (char c : s) {
|
||||
if (c == '\\' || c == '"') out.push_back('\\');
|
||||
out.push_back(c == '\n' ? ' ' : c);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Slo::Slo(::molo::common::Clock& clock, ::molo::common::Observer& observer)
|
||||
: Slo(clock, observer, Config{}) {}
|
||||
|
||||
Slo::Slo(::molo::common::Clock& clock, ::molo::common::Observer& observer, Config cfg)
|
||||
: clock_(clock), observer_(observer), cfg_(cfg) {
|
||||
if (cfg_.target_ratio < 0.0) cfg_.target_ratio = 0.0;
|
||||
if (cfg_.target_ratio > 1.0) cfg_.target_ratio = 1.0;
|
||||
if (cfg_.window_ms <= 0) cfg_.window_ms = 3600000;
|
||||
}
|
||||
|
||||
void Slo::record(const std::string& route, bool ok, const std::string& reason) {
|
||||
const std::string r = trunc(route);
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
auto it = routes_.find(r);
|
||||
if (it == routes_.end()) {
|
||||
if (routes_.size() >= ::molo::operability::Caps::kSloMaxRoutes) return;
|
||||
it = routes_.emplace(r, RouteData{}).first;
|
||||
}
|
||||
RouteData& data = it->second;
|
||||
Sample s;
|
||||
s.at_ms = clock_.steady_ms();
|
||||
s.ok = ok;
|
||||
data.samples.push_back(s);
|
||||
while (data.samples.size() > ::molo::operability::Caps::kSloMaxSamplesPerRoute) {
|
||||
data.samples.pop_front();
|
||||
}
|
||||
if (!ok) {
|
||||
bad_[{r, trunc(reason)}] += 1;
|
||||
observer_.counter("slo_bad_events_total", 1);
|
||||
} else {
|
||||
observer_.counter("slo_good_events_total", 1);
|
||||
}
|
||||
}
|
||||
|
||||
double Slo::budget_remaining(const std::string& route) {
|
||||
const std::string r = trunc(route);
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
auto it = routes_.find(r);
|
||||
if (it == routes_.end()) return 1.0;
|
||||
prune_locked(it->second, clock_.steady_ms() - cfg_.window_ms);
|
||||
const auto& samples = it->second.samples;
|
||||
if (samples.empty()) return 1.0;
|
||||
std::size_t errors = 0;
|
||||
for (const auto& s : samples) {
|
||||
if (!s.ok) ++errors;
|
||||
}
|
||||
const double budgeted = 1.0 - cfg_.target_ratio;
|
||||
const auto allowed =
|
||||
static_cast<std::size_t>(static_cast<double>(samples.size()) * budgeted + 0.999);
|
||||
if (allowed == 0) return errors == 0 ? 1.0 : 0.0;
|
||||
if (errors >= allowed) return 0.0;
|
||||
return 1.0 - static_cast<double>(errors) / static_cast<double>(allowed);
|
||||
}
|
||||
|
||||
double Slo::burn_rate(const std::string& route, std::int64_t window_ms) {
|
||||
const std::string r = trunc(route);
|
||||
if (window_ms <= 0) return 0.0;
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
auto it = routes_.find(r);
|
||||
if (it == routes_.end()) return 0.0;
|
||||
const std::int64_t cutoff = clock_.steady_ms() - window_ms;
|
||||
std::size_t total = 0;
|
||||
std::size_t errors = 0;
|
||||
for (const auto& s : it->second.samples) {
|
||||
if (s.at_ms < cutoff) continue;
|
||||
++total;
|
||||
if (!s.ok) ++errors;
|
||||
}
|
||||
if (total == 0) return 0.0;
|
||||
const double budgeted = 1.0 - cfg_.target_ratio;
|
||||
if (budgeted <= 0.0) return errors == 0 ? 0.0 : 1000000.0;
|
||||
return (static_cast<double>(errors) / static_cast<double>(total)) / budgeted;
|
||||
}
|
||||
|
||||
std::int64_t Slo::bad_events(const std::string& route, const std::string& reason) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
auto it = bad_.find({trunc(route), trunc(reason)});
|
||||
return it == bad_.end() ? 0 : it->second;
|
||||
}
|
||||
|
||||
std::string Slo::render_prometheus() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
const std::int64_t now = clock_.steady_ms();
|
||||
std::string out;
|
||||
out += "# HELP molo_slo_budget_remaining_ratio Fraction of error budget left.\n";
|
||||
out += "# TYPE molo_slo_budget_remaining_ratio gauge\n";
|
||||
out += "# HELP molo_slo_burn_rate Error ratio over budgeted ratio.\n";
|
||||
out += "# TYPE molo_slo_burn_rate gauge\n";
|
||||
out += "# HELP molo_slo_bad_events_total Bad events by route and reason.\n";
|
||||
out += "# TYPE molo_slo_bad_events_total counter\n";
|
||||
constexpr std::int64_t kWindows[2] = {300000, 3600000};
|
||||
const char* kNames[2] = {"5m", "1h"};
|
||||
char buf[64];
|
||||
for (auto& kv : routes_) {
|
||||
prune_locked(kv.second, now - cfg_.window_ms);
|
||||
const auto& samples = kv.second.samples;
|
||||
std::size_t errors = 0;
|
||||
for (const auto& s : samples) {
|
||||
if (!s.ok) ++errors;
|
||||
}
|
||||
double remaining = 1.0;
|
||||
const double budgeted = 1.0 - cfg_.target_ratio;
|
||||
const auto allowed =
|
||||
static_cast<std::size_t>(static_cast<double>(samples.size()) * budgeted + 0.999);
|
||||
if (!samples.empty()) {
|
||||
if (allowed == 0) {
|
||||
remaining = errors == 0 ? 1.0 : 0.0;
|
||||
} else if (errors >= allowed) {
|
||||
remaining = 0.0;
|
||||
} else {
|
||||
remaining = 1.0 - static_cast<double>(errors) / static_cast<double>(allowed);
|
||||
}
|
||||
}
|
||||
std::snprintf(buf, sizeof(buf), "%.6f", remaining);
|
||||
out += "molo_slo_budget_remaining_ratio{route=\"" + escape_label(kv.first) + "\"} ";
|
||||
out += buf;
|
||||
out += "\n";
|
||||
for (int w = 0; w < 2; ++w) {
|
||||
std::size_t total = 0;
|
||||
std::size_t werr = 0;
|
||||
const std::int64_t cutoff = now - kWindows[w];
|
||||
for (const auto& s : samples) {
|
||||
if (s.at_ms < cutoff) continue;
|
||||
++total;
|
||||
if (!s.ok) ++werr;
|
||||
}
|
||||
double burn = 0.0;
|
||||
if (total > 0) {
|
||||
if (budgeted <= 0.0) {
|
||||
burn = werr == 0 ? 0.0 : 1000000.0;
|
||||
} else {
|
||||
burn = (static_cast<double>(werr) / static_cast<double>(total)) / budgeted;
|
||||
}
|
||||
}
|
||||
std::snprintf(buf, sizeof(buf), "%.6f", burn);
|
||||
out += "molo_slo_burn_rate{route=\"" + escape_label(kv.first) + "\",window=\"" + kNames[w] +
|
||||
"\"} ";
|
||||
out += buf;
|
||||
out += "\n";
|
||||
}
|
||||
}
|
||||
for (const auto& kv : bad_) {
|
||||
out += "molo_slo_bad_events_total{route=\"" + escape_label(kv.first.first) + "\",reason=\"" +
|
||||
escape_label(kv.first.second) + "\"} " + std::to_string(kv.second) + "\n";
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
void Slo::prune_locked(RouteData& data, std::int64_t cutoff) {
|
||||
while (!data.samples.empty() && data.samples.front().at_ms < cutoff) data.samples.pop_front();
|
||||
}
|
||||
|
||||
} // namespace molo::resilience
|
||||
@@ -0,0 +1,63 @@
|
||||
// Error budget tracker. Records good and bad events per route template and
|
||||
// exposes the SLO series: molo_slo_budget_remaining_ratio,
|
||||
// molo_slo_burn_rate{window} and molo_slo_bad_events_total{route,reason}.
|
||||
// Production scrapes render_prometheus next to the registry exporter.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <deque>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
namespace molo::common {
|
||||
class Clock;
|
||||
class Observer;
|
||||
} // namespace molo::common
|
||||
|
||||
namespace molo::resilience {
|
||||
|
||||
class Slo {
|
||||
public:
|
||||
struct Config {
|
||||
double target_ratio = 0.999;
|
||||
std::int64_t window_ms = 3600000;
|
||||
};
|
||||
|
||||
Slo(::molo::common::Clock& clock, ::molo::common::Observer& observer);
|
||||
Slo(::molo::common::Clock& clock, ::molo::common::Observer& observer, Config cfg);
|
||||
|
||||
Slo(const Slo&) = delete;
|
||||
Slo& operator=(const Slo&) = delete;
|
||||
|
||||
void record(const std::string& route, bool ok, const std::string& reason = "");
|
||||
|
||||
// Fraction of the error budget left in the window, 0 to 1.
|
||||
double budget_remaining(const std::string& route);
|
||||
// Observed error ratio divided by budgeted ratio over window_ms.
|
||||
double burn_rate(const std::string& route, std::int64_t window_ms);
|
||||
std::int64_t bad_events(const std::string& route, const std::string& reason);
|
||||
|
||||
std::string render_prometheus();
|
||||
|
||||
private:
|
||||
struct Sample {
|
||||
std::int64_t at_ms = 0;
|
||||
bool ok = true;
|
||||
};
|
||||
struct RouteData {
|
||||
std::deque<Sample> samples;
|
||||
};
|
||||
|
||||
void prune_locked(RouteData& data, std::int64_t cutoff);
|
||||
|
||||
::molo::common::Clock& clock_;
|
||||
::molo::common::Observer& observer_;
|
||||
Config cfg_;
|
||||
mutable std::mutex mutex_;
|
||||
std::map<std::string, RouteData> routes_;
|
||||
std::map<std::pair<std::string, std::string>, std::int64_t> bad_;
|
||||
};
|
||||
|
||||
} // namespace molo::resilience
|
||||
Reference in New Issue
Block a user