409 lines
12 KiB
C++
409 lines
12 KiB
C++
#pragma once
|
|||
|
|
|
||
|
|
// Resilience, delivery, supply chain, clients and data-depth skeletons for the
|
||
|
|
// living example (namespace molo::example::fw). Every class is a Lab: a small
|
||
|
|
// self-contained state machine with injectable time, a bounded log, and a
|
||
|
|
// uniform rows() view that the demo pages and the debug snapshot render.
|
||
|
|
// Gaps are listed per class next to the real module in src/.
|
||
|
|
|
||
|
|
#include <condition_variable>
|
||
|
|
#include <deque>
|
||
|
|
#include <functional>
|
||
|
|
#include <map>
|
||
|
|
#include <memory>
|
||
|
|
#include <mutex>
|
||
|
|
#include <string>
|
||
|
|
#include <utility>
|
||
|
|
#include <vector>
|
||
|
|
|
||
|
|
namespace molo::example::fw {
|
||
|
|
|
||
|
|
namespace labcaps {
|
||
|
|
constexpr std::size_t kLogLines = 20; // bounded log per lab
|
||
|
|
constexpr long kMaxAdvanceMs = 3600000; // one clock step, 1 hour
|
||
|
|
constexpr int kMaxThreads = 16; // singleflight callers
|
||
|
|
constexpr std::size_t kMaxEntries = 5; // ttl store capacity (LRU)
|
||
|
|
constexpr std::size_t kMaxAdvisories = 8; // supplychain advisory db
|
||
|
|
constexpr int kMaxAttempts = 5; // client retry ceiling
|
||
|
|
constexpr int kMaxFaults = 5; // chaos blast radius (faults)
|
||
|
|
constexpr int kMaxPercent = 50; // chaos blast radius (percent)
|
||
|
|
} // namespace labcaps
|
||
|
|
|
||
|
|
using LabRows = std::vector<std::pair<std::string, std::string>>;
|
||
|
|
|
||
|
|
// Injectable time so every state machine is deterministic in tests.
|
||
|
|
class ManualClock {
|
||
|
|
public:
|
||
|
|
long now() const { return now_; }
|
||
|
|
void advance(long ms);
|
||
|
|
|
||
|
|
private:
|
||
|
|
long now_ = 0;
|
||
|
|
};
|
||
|
|
|
||
|
|
std::string fnv_hex(const std::string& data); // 64 bit FNV-1a, hex
|
||
|
|
std::string base64_encode(const std::string& data);
|
||
|
|
|
||
|
|
class Lab {
|
||
|
|
public:
|
||
|
|
virtual ~Lab() = default;
|
||
|
|
virtual std::string name() const = 0;
|
||
|
|
virtual LabRows rows() const = 0;
|
||
|
|
std::vector<std::string> log() const;
|
||
|
|
std::string json() const; // {"name":..,"state":{..},"log":[..]}
|
||
|
|
|
||
|
|
protected:
|
||
|
|
void note(std::string line); // caller holds mu_
|
||
|
|
mutable std::recursive_mutex mu_;
|
||
|
|
|
||
|
|
private:
|
||
|
|
std::deque<std::string> log_;
|
||
|
|
};
|
||
|
|
|
||
|
|
// src/operability/resilience/breaker.h
|
||
|
|
class CircuitBreaker : public Lab {
|
||
|
|
public:
|
||
|
|
enum class State { Closed, Open, HalfOpen };
|
||
|
|
CircuitBreaker(std::shared_ptr<ManualClock> clock, int threshold = 3,
|
||
|
|
long open_ms = 30000);
|
||
|
|
// Records one call. Returns false when the breaker rejected it (open).
|
||
|
|
bool call(bool success);
|
||
|
|
void tick(long ms);
|
||
|
|
State state() const;
|
||
|
|
std::string name() const override { return "breaker"; }
|
||
|
|
LabRows rows() const override;
|
||
|
|
|
||
|
|
private:
|
||
|
|
State effective() const;
|
||
|
|
std::shared_ptr<ManualClock> clock_;
|
||
|
|
int threshold_;
|
||
|
|
long open_ms_;
|
||
|
|
State state_ = State::Closed;
|
||
|
|
int failures_ = 0;
|
||
|
|
long opened_at_ = 0;
|
||
|
|
int rejected_ = 0;
|
||
|
|
};
|
||
|
|
std::string breaker_state_name(CircuitBreaker::State s);
|
||
|
|
|
||
|
|
// src/operability/resilience/bulkhead.h
|
||
|
|
class Bulkhead : public Lab {
|
||
|
|
public:
|
||
|
|
enum class Admission { Admitted, Queued, Rejected };
|
||
|
|
Bulkhead(int max_active = 2, int max_queue = 1);
|
||
|
|
Admission acquire();
|
||
|
|
bool release(); // false when nothing to release
|
||
|
|
std::string name() const override { return "bulkhead"; }
|
||
|
|
LabRows rows() const override;
|
||
|
|
|
||
|
|
private:
|
||
|
|
int max_active_, max_queue_, active_ = 0, queued_ = 0, rejected_ = 0;
|
||
|
|
};
|
||
|
|
|
||
|
|
// src/operability/resilience/shedder.h
|
||
|
|
class Shedder : public Lab {
|
||
|
|
public:
|
||
|
|
explicit Shedder(int limit = 3) : limit_(limit) {}
|
||
|
|
// low priority is shed at 2/3 of the limit, high only at the limit.
|
||
|
|
bool enter(bool high);
|
||
|
|
bool leave();
|
||
|
|
std::string name() const override { return "shedder"; }
|
||
|
|
LabRows rows() const override;
|
||
|
|
|
||
|
|
private:
|
||
|
|
int limit_, inflight_ = 0, shed_low_ = 0, shed_high_ = 0;
|
||
|
|
};
|
||
|
|
|
||
|
|
// src/operability/resilience/deadline.h
|
||
|
|
class DeadlineLab : public Lab {
|
||
|
|
public:
|
||
|
|
static constexpr long kHopCostMs = 30;
|
||
|
|
// Walks `hops` downstream calls against one propagated budget.
|
||
|
|
// Returns the hop that expired (1 based) or 0 when all finished.
|
||
|
|
int run(long budget_ms, int hops);
|
||
|
|
std::string name() const override { return "deadline"; }
|
||
|
|
LabRows rows() const override;
|
||
|
|
|
||
|
|
private:
|
||
|
|
long budget_ms_ = 0, remaining_ms_ = 0;
|
||
|
|
int hops_ = 0, expired_hop_ = 0;
|
||
|
|
bool ran_ = false;
|
||
|
|
};
|
||
|
|
|
||
|
|
// src/operability/resilience/slo.h
|
||
|
|
class SloBudget : public Lab {
|
||
|
|
public:
|
||
|
|
static constexpr std::size_t kWindow = 100; // ring of the last 100 events
|
||
|
|
SloBudget(int target_permille = 950) : target_(target_permille) {}
|
||
|
|
void record(bool good);
|
||
|
|
void reset();
|
||
|
|
long budget_remaining() const; // allowed bad events minus actual bad
|
||
|
|
std::string name() const override { return "slo"; }
|
||
|
|
LabRows rows() const override;
|
||
|
|
|
||
|
|
private:
|
||
|
|
int target_;
|
||
|
|
std::deque<bool> window_;
|
||
|
|
};
|
||
|
|
|
||
|
|
// src/operability/chaos/experiment.h, injector.h
|
||
|
|
class ChaosLab : public Lab {
|
||
|
|
public:
|
||
|
|
static constexpr int kCalls = 10; // simulated calls per experiment
|
||
|
|
struct Result {
|
||
|
|
int percent = 0, max_faults = 0, planned = 0, injected = 0;
|
||
|
|
bool armed = false;
|
||
|
|
};
|
||
|
|
// Deterministic: an accumulator injects a fault every 100/percent calls.
|
||
|
|
// Dry run (armed=false) plans faults but never injects them.
|
||
|
|
Result run(int percent, int max_faults, bool armed);
|
||
|
|
std::string name() const override { return "chaos"; }
|
||
|
|
LabRows rows() const override;
|
||
|
|
|
||
|
|
private:
|
||
|
|
Result last_;
|
||
|
|
int runs_ = 0, total_injected_ = 0;
|
||
|
|
};
|
||
|
|
|
||
|
|
// src/operability/deploy/rolling.h, blue_green.h, lifecycle.h
|
||
|
|
class DeployLab : public Lab {
|
||
|
|
public:
|
||
|
|
enum class Phase { Idle, InProgress, Done, RolledBack };
|
||
|
|
DeployLab();
|
||
|
|
bool start(const std::string& strategy, const std::string& version);
|
||
|
|
bool step();
|
||
|
|
bool rollback();
|
||
|
|
Phase phase() const;
|
||
|
|
std::string name() const override { return "deploy"; }
|
||
|
|
LabRows rows() const override;
|
||
|
|
|
||
|
|
private:
|
||
|
|
std::vector<std::string> fleet_, previous_;
|
||
|
|
std::string strategy_ = "none", target_;
|
||
|
|
int green_stage_ = 0; // blue-green: 0 prepare, 1 switched
|
||
|
|
Phase phase_ = Phase::Idle;
|
||
|
|
};
|
||
|
|
std::string deploy_phase_name(DeployLab::Phase p);
|
||
|
|
|
||
|
|
// src/operability/coord/lease.h, election.h, dist_lock.h, fence.h
|
||
|
|
class CoordLab : public Lab {
|
||
|
|
public:
|
||
|
|
struct Grant {
|
||
|
|
bool ok = false;
|
||
|
|
long token = 0;
|
||
|
|
std::string holder;
|
||
|
|
};
|
||
|
|
explicit CoordLab(std::shared_ptr<ManualClock> clock, long ttl_ms = 10000);
|
||
|
|
Grant acquire(const std::string& key, const std::string& owner);
|
||
|
|
bool release(const std::string& key, const std::string& owner);
|
||
|
|
// Fenced write: accepted only when token >= the highest token seen.
|
||
|
|
bool write(const std::string& key, long token);
|
||
|
|
void tick(long ms);
|
||
|
|
static bool known_key(const std::string& key);
|
||
|
|
std::string name() const override { return "coord"; }
|
||
|
|
LabRows rows() const override;
|
||
|
|
|
||
|
|
private:
|
||
|
|
struct Slot {
|
||
|
|
std::string holder;
|
||
|
|
long expires = 0, token = 0, fence = 0;
|
||
|
|
};
|
||
|
|
std::shared_ptr<ManualClock> clock_;
|
||
|
|
long ttl_ms_;
|
||
|
|
std::map<std::string, Slot> slots_;
|
||
|
|
};
|
||
|
|
|
||
|
|
// src/operability/debug/endpoint.h, snapshot.h
|
||
|
|
class DebugRegistry {
|
||
|
|
public:
|
||
|
|
void add(std::shared_ptr<Lab> lab);
|
||
|
|
std::string snapshot() const; // {"labs":{name:{..}}}
|
||
|
|
std::size_t size() const { return labs_.size(); }
|
||
|
|
|
||
|
|
private:
|
||
|
|
std::vector<std::shared_ptr<Lab>> labs_;
|
||
|
|
};
|
||
|
|
|
||
|
|
// src/supplychain/sbom, policy, attest
|
||
|
|
class SupplyChainLab : public Lab {
|
||
|
|
public:
|
||
|
|
struct Component {
|
||
|
|
std::string name, version, license;
|
||
|
|
};
|
||
|
|
SupplyChainLab();
|
||
|
|
bool add_advisory(const std::string& component, const std::string& version);
|
||
|
|
void tamper();
|
||
|
|
void reset();
|
||
|
|
// True when `digest` equals the attested and the current SBOM digest.
|
||
|
|
bool verify(const std::string& digest);
|
||
|
|
std::string digest() const;
|
||
|
|
std::string attested() const { return attested_; }
|
||
|
|
bool gate() const;
|
||
|
|
std::string cyclonedx() const;
|
||
|
|
std::string name() const override { return "supplychain"; }
|
||
|
|
LabRows rows() const override;
|
||
|
|
|
||
|
|
private:
|
||
|
|
bool pins_ok() const;
|
||
|
|
bool advisories_ok() const;
|
||
|
|
std::vector<Component> components_;
|
||
|
|
std::map<std::string, std::string> pins_;
|
||
|
|
std::vector<std::pair<std::string, std::string>> advisories_;
|
||
|
|
std::string attested_;
|
||
|
|
int verified_ = 0; // 0 none, 1 ok, -1 failed
|
||
|
|
};
|
||
|
|
|
||
|
|
// src/clients/retry.h
|
||
|
|
class ClientRetryLab : public Lab {
|
||
|
|
public:
|
||
|
|
struct Outcome {
|
||
|
|
int attempts = 0, status = 0;
|
||
|
|
long waited_ms = 0;
|
||
|
|
bool gave_up = false;
|
||
|
|
};
|
||
|
|
// The in-process transport fails the first `failures` calls with `status`.
|
||
|
|
// 5xx is retried with capped exponential backoff; 4xx never is.
|
||
|
|
Outcome run(int failures, int max_attempts, int status);
|
||
|
|
std::string name() const override { return "client-retry"; }
|
||
|
|
LabRows rows() const override;
|
||
|
|
|
||
|
|
private:
|
||
|
|
static constexpr long kBaseMs = 100, kCapMs = 800;
|
||
|
|
Outcome last_;
|
||
|
|
int runs_ = 0;
|
||
|
|
};
|
||
|
|
|
||
|
|
// src/clients/cookies.h, pool.h, proxy.h, auth.h
|
||
|
|
class ClientSessionLab : public Lab {
|
||
|
|
public:
|
||
|
|
ClientSessionLab();
|
||
|
|
static bool valid_proxy(const std::string& proxy);
|
||
|
|
// One request through the in-process transport; returns the request headers.
|
||
|
|
std::map<std::string, std::string> request(const std::string& auth,
|
||
|
|
const std::string& proxy);
|
||
|
|
std::string name() const override { return "client-session"; }
|
||
|
|
LabRows rows() const override;
|
||
|
|
|
||
|
|
private:
|
||
|
|
std::map<std::string, std::string> jar_;
|
||
|
|
int created_ = 0, reused_ = 0, idle_ = 0, requests_ = 0;
|
||
|
|
std::string proxy_ = "direct", auth_ = "none";
|
||
|
|
};
|
||
|
|
|
||
|
|
// src/data/orm/migration.hpp, migrator.hpp, healer.hpp
|
||
|
|
class MigrationLab : public Lab {
|
||
|
|
public:
|
||
|
|
struct Step {
|
||
|
|
std::string id, sql, table;
|
||
|
|
};
|
||
|
|
MigrationLab();
|
||
|
|
// check: what apply would do, nothing changes.
|
||
|
|
std::vector<std::string> check() const;
|
||
|
|
int apply(int limit);
|
||
|
|
void drop_table(const std::string& table); // simulated drift
|
||
|
|
int heal(); // re-apply drifted creates
|
||
|
|
std::vector<std::string> drift() const;
|
||
|
|
std::string name() const override { return "migrations"; }
|
||
|
|
LabRows rows() const override;
|
||
|
|
|
||
|
|
private:
|
||
|
|
std::vector<Step> steps_;
|
||
|
|
std::vector<std::string> applied_;
|
||
|
|
std::map<std::string, std::string> tables_; // table -> columns
|
||
|
|
};
|
||
|
|
|
||
|
|
// src/data/transaction.hpp
|
||
|
|
class TxLab : public Lab {
|
||
|
|
public:
|
||
|
|
TxLab();
|
||
|
|
// Returns "committed", or the rollback reason.
|
||
|
|
std::string transfer(const std::string& from, const std::string& to,
|
||
|
|
long amount, bool fail_after_debit);
|
||
|
|
long balance(const std::string& who) const;
|
||
|
|
std::string name() const override { return "transaction"; }
|
||
|
|
LabRows rows() const override;
|
||
|
|
|
||
|
|
private:
|
||
|
|
std::map<std::string, long> accounts_;
|
||
|
|
int commits_ = 0, rollbacks_ = 0;
|
||
|
|
};
|
||
|
|
|
||
|
|
// src/data/orm/query.hpp, relations.hpp
|
||
|
|
class QueryLab : public Lab {
|
||
|
|
public:
|
||
|
|
struct Result {
|
||
|
|
std::string sql;
|
||
|
|
std::vector<std::map<std::string, std::string>> rows;
|
||
|
|
};
|
||
|
|
QueryLab();
|
||
|
|
// Builds "SELECT ... FROM posts [WHERE user_id = ?] ORDER BY .. LIMIT n"
|
||
|
|
// and executes it against the in-memory users/posts tables (has-many).
|
||
|
|
Result run(long user_id, const std::string& order, int limit);
|
||
|
|
std::string name() const override { return "query"; }
|
||
|
|
LabRows rows() const override;
|
||
|
|
|
||
|
|
private:
|
||
|
|
std::vector<std::map<std::string, std::string>> users_, posts_;
|
||
|
|
Result last_;
|
||
|
|
};
|
||
|
|
|
||
|
|
// src/data/cache/singleflight.hpp
|
||
|
|
class SingleFlight {
|
||
|
|
public:
|
||
|
|
// Concurrent callers of one key share a single loader execution.
|
||
|
|
std::string run(const std::string& key,
|
||
|
|
const std::function<std::string()>& loader);
|
||
|
|
int waiters(const std::string& key) const;
|
||
|
|
|
||
|
|
private:
|
||
|
|
struct Call {
|
||
|
|
bool done = false;
|
||
|
|
int waiters = 0;
|
||
|
|
std::string value;
|
||
|
|
};
|
||
|
|
mutable std::mutex mu_;
|
||
|
|
std::condition_variable_any cv_;
|
||
|
|
std::map<std::string, std::shared_ptr<Call>> calls_;
|
||
|
|
};
|
||
|
|
|
||
|
|
class SingleFlightLab : public Lab {
|
||
|
|
public:
|
||
|
|
SingleFlightLab();
|
||
|
|
// Starts `callers` threads on one key; returns loader executions.
|
||
|
|
int run(int callers);
|
||
|
|
std::string name() const override { return "singleflight"; }
|
||
|
|
LabRows rows() const override;
|
||
|
|
|
||
|
|
private:
|
||
|
|
std::unique_ptr<SingleFlight> flight_;
|
||
|
|
int callers_ = 0, loads_ = 0, runs_ = 0;
|
||
|
|
};
|
||
|
|
|
||
|
|
// src/data/kv/memory.hpp, src/data/cache/policy.hpp, src/data/pubsub/broker.hpp
|
||
|
|
class TtlLab : public Lab {
|
||
|
|
public:
|
||
|
|
explicit TtlLab(std::shared_ptr<ManualClock> clock);
|
||
|
|
void set(const std::string& key, const std::string& value, long ttl_s);
|
||
|
|
// Returns true and fills value on a live hit.
|
||
|
|
bool get(const std::string& key, std::string& value);
|
||
|
|
void tick(long ms);
|
||
|
|
std::size_t size() const;
|
||
|
|
std::vector<std::string> events() const; // pubsub topic "kv.events"
|
||
|
|
std::string name() const override { return "ttl"; }
|
||
|
|
LabRows rows() const override;
|
||
|
|
|
||
|
|
private:
|
||
|
|
struct Entry {
|
||
|
|
std::string value;
|
||
|
|
long expires = 0, touched = 0;
|
||
|
|
};
|
||
|
|
void sweep();
|
||
|
|
void publish(const std::string& event);
|
||
|
|
std::shared_ptr<ManualClock> clock_;
|
||
|
|
std::map<std::string, Entry> entries_;
|
||
|
|
std::deque<std::string> events_;
|
||
|
|
long seq_ = 0;
|
||
|
|
int hits_ = 0, misses_ = 0;
|
||
|
|
};
|
||
|
|
|
||
|
|
} // namespace molo::example::fw
|