#pragma once // Resilience, delivery, supply chain, clients and data-depth skeletons for the // living example (namespace molo::example::fw). Every class is a Lab: a small // self-contained state machine with injectable time, a bounded log, and a // uniform rows() view that the demo pages and the debug snapshot render. // Gaps are listed per class next to the real module in src/. #include #include #include #include #include #include #include #include #include namespace molo::example::fw { namespace labcaps { constexpr std::size_t kLogLines = 20; // bounded log per lab constexpr long kMaxAdvanceMs = 3600000; // one clock step, 1 hour constexpr int kMaxThreads = 16; // singleflight callers constexpr std::size_t kMaxEntries = 5; // ttl store capacity (LRU) constexpr std::size_t kMaxAdvisories = 8; // supplychain advisory db constexpr int kMaxAttempts = 5; // client retry ceiling constexpr int kMaxFaults = 5; // chaos blast radius (faults) constexpr int kMaxPercent = 50; // chaos blast radius (percent) } // namespace labcaps using LabRows = std::vector>; // Injectable time so every state machine is deterministic in tests. class ManualClock { public: long now() const { return now_; } void advance(long ms); private: long now_ = 0; }; std::string fnv_hex(const std::string& data); // 64 bit FNV-1a, hex std::string base64_encode(const std::string& data); class Lab { public: virtual ~Lab() = default; virtual std::string name() const = 0; virtual LabRows rows() const = 0; std::vector log() const; std::string json() const; // {"name":..,"state":{..},"log":[..]} protected: void note(std::string line); // caller holds mu_ mutable std::recursive_mutex mu_; private: std::deque log_; }; // src/operability/resilience/breaker.h class CircuitBreaker : public Lab { public: enum class State { Closed, Open, HalfOpen }; CircuitBreaker(std::shared_ptr clock, int threshold = 3, long open_ms = 30000); // Records one call. Returns false when the breaker rejected it (open). bool call(bool success); void tick(long ms); State state() const; std::string name() const override { return "breaker"; } LabRows rows() const override; private: State effective() const; std::shared_ptr clock_; int threshold_; long open_ms_; State state_ = State::Closed; int failures_ = 0; long opened_at_ = 0; int rejected_ = 0; }; std::string breaker_state_name(CircuitBreaker::State s); // src/operability/resilience/bulkhead.h class Bulkhead : public Lab { public: enum class Admission { Admitted, Queued, Rejected }; Bulkhead(int max_active = 2, int max_queue = 1); Admission acquire(); bool release(); // false when nothing to release std::string name() const override { return "bulkhead"; } LabRows rows() const override; private: int max_active_, max_queue_, active_ = 0, queued_ = 0, rejected_ = 0; }; // src/operability/resilience/shedder.h class Shedder : public Lab { public: explicit Shedder(int limit = 3) : limit_(limit) {} // low priority is shed at 2/3 of the limit, high only at the limit. bool enter(bool high); bool leave(); std::string name() const override { return "shedder"; } LabRows rows() const override; private: int limit_, inflight_ = 0, shed_low_ = 0, shed_high_ = 0; }; // src/operability/resilience/deadline.h class DeadlineLab : public Lab { public: static constexpr long kHopCostMs = 30; // Walks `hops` downstream calls against one propagated budget. // Returns the hop that expired (1 based) or 0 when all finished. int run(long budget_ms, int hops); std::string name() const override { return "deadline"; } LabRows rows() const override; private: long budget_ms_ = 0, remaining_ms_ = 0; int hops_ = 0, expired_hop_ = 0; bool ran_ = false; }; // src/operability/resilience/slo.h class SloBudget : public Lab { public: static constexpr std::size_t kWindow = 100; // ring of the last 100 events SloBudget(int target_permille = 950) : target_(target_permille) {} void record(bool good); void reset(); long budget_remaining() const; // allowed bad events minus actual bad std::string name() const override { return "slo"; } LabRows rows() const override; private: int target_; std::deque window_; }; // src/operability/chaos/experiment.h, injector.h class ChaosLab : public Lab { public: static constexpr int kCalls = 10; // simulated calls per experiment struct Result { int percent = 0, max_faults = 0, planned = 0, injected = 0; bool armed = false; }; // Deterministic: an accumulator injects a fault every 100/percent calls. // Dry run (armed=false) plans faults but never injects them. Result run(int percent, int max_faults, bool armed); std::string name() const override { return "chaos"; } LabRows rows() const override; private: Result last_; int runs_ = 0, total_injected_ = 0; }; // src/operability/deploy/rolling.h, blue_green.h, lifecycle.h class DeployLab : public Lab { public: enum class Phase { Idle, InProgress, Done, RolledBack }; DeployLab(); bool start(const std::string& strategy, const std::string& version); bool step(); bool rollback(); Phase phase() const; std::string name() const override { return "deploy"; } LabRows rows() const override; private: std::vector fleet_, previous_; std::string strategy_ = "none", target_; int green_stage_ = 0; // blue-green: 0 prepare, 1 switched Phase phase_ = Phase::Idle; }; std::string deploy_phase_name(DeployLab::Phase p); // src/operability/coord/lease.h, election.h, dist_lock.h, fence.h class CoordLab : public Lab { public: struct Grant { bool ok = false; long token = 0; std::string holder; }; explicit CoordLab(std::shared_ptr clock, long ttl_ms = 10000); Grant acquire(const std::string& key, const std::string& owner); bool release(const std::string& key, const std::string& owner); // Fenced write: accepted only when token >= the highest token seen. bool write(const std::string& key, long token); void tick(long ms); static bool known_key(const std::string& key); std::string name() const override { return "coord"; } LabRows rows() const override; private: struct Slot { std::string holder; long expires = 0, token = 0, fence = 0; }; std::shared_ptr clock_; long ttl_ms_; std::map slots_; }; // src/operability/debug/endpoint.h, snapshot.h class DebugRegistry { public: void add(std::shared_ptr lab); std::string snapshot() const; // {"labs":{name:{..}}} std::size_t size() const { return labs_.size(); } private: std::vector> labs_; }; // src/supplychain/sbom, policy, attest class SupplyChainLab : public Lab { public: struct Component { std::string name, version, license; }; SupplyChainLab(); bool add_advisory(const std::string& component, const std::string& version); void tamper(); void reset(); // True when `digest` equals the attested and the current SBOM digest. bool verify(const std::string& digest); std::string digest() const; std::string attested() const { return attested_; } bool gate() const; std::string cyclonedx() const; std::string name() const override { return "supplychain"; } LabRows rows() const override; private: bool pins_ok() const; bool advisories_ok() const; std::vector components_; std::map pins_; std::vector> advisories_; std::string attested_; int verified_ = 0; // 0 none, 1 ok, -1 failed }; // src/clients/retry.h class ClientRetryLab : public Lab { public: struct Outcome { int attempts = 0, status = 0; long waited_ms = 0; bool gave_up = false; }; // The in-process transport fails the first `failures` calls with `status`. // 5xx is retried with capped exponential backoff; 4xx never is. Outcome run(int failures, int max_attempts, int status); std::string name() const override { return "client-retry"; } LabRows rows() const override; private: static constexpr long kBaseMs = 100, kCapMs = 800; Outcome last_; int runs_ = 0; }; // src/clients/cookies.h, pool.h, proxy.h, auth.h class ClientSessionLab : public Lab { public: ClientSessionLab(); static bool valid_proxy(const std::string& proxy); // One request through the in-process transport; returns the request headers. std::map request(const std::string& auth, const std::string& proxy); std::string name() const override { return "client-session"; } LabRows rows() const override; private: std::map jar_; int created_ = 0, reused_ = 0, idle_ = 0, requests_ = 0; std::string proxy_ = "direct", auth_ = "none"; }; // src/data/orm/migration.hpp, migrator.hpp, healer.hpp class MigrationLab : public Lab { public: struct Step { std::string id, sql, table; }; MigrationLab(); // check: what apply would do, nothing changes. std::vector check() const; int apply(int limit); void drop_table(const std::string& table); // simulated drift int heal(); // re-apply drifted creates std::vector drift() const; std::string name() const override { return "migrations"; } LabRows rows() const override; private: std::vector steps_; std::vector applied_; std::map tables_; // table -> columns }; // src/data/transaction.hpp class TxLab : public Lab { public: TxLab(); // Returns "committed", or the rollback reason. std::string transfer(const std::string& from, const std::string& to, long amount, bool fail_after_debit); long balance(const std::string& who) const; std::string name() const override { return "transaction"; } LabRows rows() const override; private: std::map accounts_; int commits_ = 0, rollbacks_ = 0; }; // src/data/orm/query.hpp, relations.hpp class QueryLab : public Lab { public: struct Result { std::string sql; std::vector> rows; }; QueryLab(); // Builds "SELECT ... FROM posts [WHERE user_id = ?] ORDER BY .. LIMIT n" // and executes it against the in-memory users/posts tables (has-many). Result run(long user_id, const std::string& order, int limit); std::string name() const override { return "query"; } LabRows rows() const override; private: std::vector> users_, posts_; Result last_; }; // src/data/cache/singleflight.hpp class SingleFlight { public: // Concurrent callers of one key share a single loader execution. std::string run(const std::string& key, const std::function& loader); int waiters(const std::string& key) const; private: struct Call { bool done = false; int waiters = 0; std::string value; }; mutable std::mutex mu_; std::condition_variable_any cv_; std::map> calls_; }; class SingleFlightLab : public Lab { public: SingleFlightLab(); // Starts `callers` threads on one key; returns loader executions. int run(int callers); std::string name() const override { return "singleflight"; } LabRows rows() const override; private: std::unique_ptr flight_; int callers_ = 0, loads_ = 0, runs_ = 0; }; // src/data/kv/memory.hpp, src/data/cache/policy.hpp, src/data/pubsub/broker.hpp class TtlLab : public Lab { public: explicit TtlLab(std::shared_ptr clock); void set(const std::string& key, const std::string& value, long ttl_s); // Returns true and fills value on a live hit. bool get(const std::string& key, std::string& value); void tick(long ms); std::size_t size() const; std::vector events() const; // pubsub topic "kv.events" std::string name() const override { return "ttl"; } LabRows rows() const override; private: struct Entry { std::string value; long expires = 0, touched = 0; }; void sweep(); void publish(const std::string& event); std::shared_ptr clock_; std::map entries_; std::deque events_; long seq_ = 0; int hits_ = 0, misses_ = 0; }; } // namespace molo::example::fw