Files
moloapp/example/app/ResilienceDemo.cpp
T
retoorandClaude Sonnet 5.5 62fdd08da8 feat(workflow): example covers security, platform, enterprise and resilience features
105 route tests, matrix crawl, spec and README updated.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-24 02:19:52 +02:00

622 lines
25 KiB
C++

#include "App.hpp"
#include <algorithm>
#include "../fw/Resilience.hpp"
#include "../fw/Router.hpp"
#include "../fw/Templates.hpp"
#include "../fw/Validate.hpp"
#include "FormUi.hpp"
// Interactive lab pages for operability, supply chain, clients and data depth.
// Each page is a LabPage: a GET that renders the lab state plus one POST route
// per button. HTML clients get the page back, JSON clients get the lab state;
// validation failures render the shared inline-error form (422).
namespace molo::example::app {
namespace {
using Params = std::map<std::string, std::string>;
const char kLabPage[] = R"LAB({% extends "base.html" %}
{% block content %}
<h1>{{ title }}</h1>
<p class="hint">{{ intro }}</p>
{% if notice %}<p class="ok" role="status">{{ notice }}</p>{% endif %}
<div class="scroll"><table>
<thead><tr><th>State</th><th>Value</th></tr></thead>
<tbody>
{% for r in rows %}
<tr><td>{{ r.k }}</td><td>{{ r.v }}</td></tr>
{% endfor %}
</tbody>
</table></div>
{% for a in actions %}
{{ a.html|safe }}
{% endfor %}
<h2>Log</h2>
<ul>
{% for l in log %}
<li>{{ l.line }}</li>
{% endfor %}
</ul>
{% endblock %}
)LAB";
// Throws the standard field error when `value` is not one of `allowed`.
void require_in(const std::string& field, const std::string& value,
const std::vector<std::string>& allowed) {
if (std::find(allowed.begin(), allowed.end(), value) != allowed.end()) return;
std::string list;
for (const auto& a : allowed) list += (list.empty() ? "" : ", ") + a;
throw fw::ValidationError({{field, "must be one of: " + list}});
}
// One button (with its inputs) on a lab page and the handler behind it.
class ActionSpec {
public:
ActionSpec(std::string sub, std::string label)
: sub_(std::move(sub)), label_(std::move(label)) {}
ActionSpec& num(const std::string& name, const std::string& label, long lo,
long hi, const std::string& def, const std::string& hint = "") {
model_.push_back({name, fw::FieldType::Integer, false, 0, 1024, lo, hi,
true, def});
form_.push_back({name, label, "number",
hint.empty() ? std::to_string(lo) + " to " + std::to_string(hi)
: hint});
defaults_[name] = def;
return *this;
}
ActionSpec& text(const std::string& name, const std::string& label,
std::size_t min, std::size_t max, const std::string& def,
const std::string& hint = "", bool required = true) {
model_.push_back({name, fw::FieldType::String, required, min, max, 0, 0,
false, required ? "" : def});
form_.push_back({name, label, "text", hint});
defaults_[name] = def;
return *this;
}
ActionSpec& run(std::function<std::string(const Params&)> fn) {
run_ = std::move(fn);
return *this;
}
ActionSpec& dynamic_defaults(std::function<Params()> fn) {
dyn_ = std::move(fn);
return *this;
}
const std::string& sub() const { return sub_; }
const std::string& label() const { return label_; }
const std::vector<FormField>& form() const { return form_; }
fw::Model model() const { return fw::Model(model_); }
Params defaults() const {
Params d = defaults_;
if (dyn_)
for (const auto& [k, v] : dyn_()) d[k] = v;
return d;
}
std::string execute(const Params& p) const { return run_ ? run_(p) : ""; }
private:
std::string sub_, label_;
std::vector<fw::Field> model_;
std::vector<FormField> form_;
Params defaults_;
std::function<Params()> dyn_;
std::function<std::string(const Params&)> run_;
};
class LabPage {
public:
LabPage(fw::Environment* env, std::string path, std::string title,
std::string intro, std::shared_ptr<fw::Lab> lab)
: env_(env),
path_(std::move(path)),
title_(std::move(title)),
intro_(std::move(intro)),
lab_(std::move(lab)) {}
ActionSpec& action(std::string sub, std::string label) {
actions_.emplace_back(std::move(sub), std::move(label));
return actions_.back();
}
void extra_rows(std::function<fw::LabRows()> fn) { extra_ = std::move(fn); }
fw::Response page(int status, const std::string& notice) const {
fw::TemplateCtx ctx;
ctx["title"] = title_;
ctx["intro"] = intro_;
ctx["notice"] = notice;
fw::TemplateList rows, acts, log;
fw::LabRows all = lab_->rows();
if (extra_)
for (auto& r : extra_()) all.push_back(std::move(r));
for (const auto& [k, v] : all) rows.push_back({{"k", k}, {"v", v}});
for (const auto& a : actions_) acts.push_back({{"html", action_html(a)}});
for (const auto& l : lab_->log()) log.push_back({{"line", l}});
ctx["rows"] = rows;
ctx["actions"] = acts;
ctx["log"] = log;
return fw::Response(status, env_->render("lab_page.html", ctx),
"text/html; charset=utf-8");
}
fw::Response invalid(const ActionSpec& a, const fw::Request& req,
const fw::ValidationError& e) const {
if (!wants_html(req)) return fw::JsonResponse(e.to_json(), 422);
FormView v;
v.title = a.label();
v.action = path_ + a.sub();
v.submit = a.label();
v.fields = a.form();
v.values = a.defaults();
for (const auto& [k, val] : fw::parse_form(req.body)) v.values[k] = val;
v.error = &e;
return fw::Response(422, env_->render("form_page.html", v.ctx()),
"text/html; charset=utf-8");
}
fw::Response handle(std::size_t i, const fw::Request& req) const {
const ActionSpec& a = actions_[i];
std::string notice;
try {
notice = a.execute(fw::Form(a.model()).bind(req));
} catch (const fw::ValidationError& e) {
return invalid(a, req, e);
}
if (!wants_html(req)) return fw::JsonResponse(lab_->json(), 200);
return page(200, notice);
}
void mount(fw::Router& router, const std::shared_ptr<LabPage>& self,
const std::string& name) {
router.get(path_, name, [self](fw::Request) { return self->page(200, ""); });
for (std::size_t i = 0; i < actions_.size(); ++i)
router.post(path_ + actions_[i].sub(), name + actions_[i].sub(),
[self, i](fw::Request req) { return self->handle(i, req); });
}
private:
std::string action_html(const ActionSpec& a) const {
std::string h = "<form method=\"post\" class=\"toolbar\" action=\"" +
fw::html_escape(path_ + a.sub()) + "\">";
Params d = a.defaults();
for (const auto& f : a.form()) {
h += "<label class=\"inline\">" + fw::html_escape(f.label) +
" <input type=\"" + f.type + "\" name=\"" + fw::html_escape(f.name) +
"\" value=\"" + fw::html_escape(d[f.name]) +
"\" style=\"max-width:9rem\" title=\"" + fw::html_escape(f.hint) +
"\"></label>";
}
return h + "<button type=\"submit\">" + fw::html_escape(a.label()) +
"</button></form>";
}
fw::Environment* env_;
std::string path_, title_, intro_;
std::shared_ptr<fw::Lab> lab_;
std::deque<ActionSpec> actions_; // deque: stable references while adding
std::function<fw::LabRows()> extra_;
};
long to_long(const Params& p, const std::string& k) {
return std::stol(p.at(k));
}
// Builds, wires and registers one lab page in the index matrix.
class Mounter {
public:
Mounter(fw::Router& router, Services& services,
std::shared_ptr<fw::DebugRegistry> registry)
: router_(router), services_(services), registry_(std::move(registry)) {}
std::shared_ptr<LabPage> page(const std::string& feature,
const std::string& path,
const std::string& title,
const std::string& intro,
std::shared_ptr<fw::Lab> lab) {
auto p = std::make_shared<LabPage>(services_.templates.get(), path, title,
intro, std::move(lab));
pending_.push_back({p, feature, path});
return p;
}
void finish() {
for (auto& e : pending_) {
e.page->mount(router_, e.page, e.path);
services_.add_feature(e.feature, e.path);
}
pending_.clear();
}
private:
struct Entry {
std::shared_ptr<LabPage> page;
std::string feature, path;
};
fw::Router& router_;
Services& services_;
std::shared_ptr<fw::DebugRegistry> registry_;
std::vector<Entry> pending_;
};
} // namespace
void register_resilience_demo(fw::Router& router, Services& services) {
services.templates->add_template("lab_page.html", kLabPage);
auto registry = std::make_shared<fw::DebugRegistry>();
Mounter m(router, services, registry);
// ---- operability: resilience -------------------------------------------
auto bclock = std::make_shared<fw::ManualClock>();
auto breaker = std::make_shared<fw::CircuitBreaker>(bclock);
registry->add(breaker);
auto p = m.page("Circuit breaker", "/operability/breaker", "Circuit breaker",
"Three failures open the breaker, calls are rejected while "
"open, 30 s of injected clock time later it goes half-open "
"and one good call closes it again.",
breaker);
p->action("/success", "Call succeeds").run([breaker](const Params&) {
return breaker->call(true) ? "call recorded" : "rejected: breaker open";
});
p->action("/failure", "Call fails").run([breaker](const Params&) {
return breaker->call(false) ? "failure recorded" : "rejected: breaker open";
});
p->action("/tick", "Advance 10 s").run([breaker](const Params&) {
breaker->tick(10000);
return "clock advanced 10 s";
});
auto bulkhead = std::make_shared<fw::Bulkhead>();
registry->add(bulkhead);
p = m.page("Bulkhead", "/operability/bulkhead", "Bulkhead",
"Two concurrent slots and a queue of one; the fourth caller is "
"rejected instead of starving the rest.",
bulkhead);
p->action("/acquire", "Take a slot").run([bulkhead](const Params&) {
switch (bulkhead->acquire()) {
case fw::Bulkhead::Admission::Admitted: return "admitted";
case fw::Bulkhead::Admission::Queued: return "queued";
default: return "rejected: compartment full";
}
});
p->action("/release", "Release a slot").run([bulkhead](const Params&) {
return bulkhead->release() ? "released" : "nothing to release";
});
auto shedder = std::make_shared<fw::Shedder>();
registry->add(shedder);
p = m.page("Load shedder", "/operability/shedder", "Load shedder",
"Limit 3 in flight. Low priority work is shed first (at 2), high "
"priority only at the limit.",
shedder);
p->action("/enter", "Start request")
.text("priority", "Priority", 1, 8, "low", "low or high", false)
.run([shedder](const Params& q) {
require_in("priority", q.at("priority"), {"low", "high"});
return shedder->enter(q.at("priority") == "high") ? "accepted" : "shed";
});
p->action("/leave", "Finish request").run([shedder](const Params&) {
return shedder->leave() ? "finished" : "nothing in flight";
});
auto deadline = std::make_shared<fw::DeadlineLab>();
registry->add(deadline);
p = m.page("Deadline propagation", "/operability/deadline",
"Deadline propagation",
"One budget travels through every hop; each hop costs 30 ms and a "
"hop that cannot finish in time is skipped.",
deadline);
p->action("/run", "Run chain")
.num("budget_ms", "Budget ms", 0, 5000, "100")
.num("hops", "Hops", 1, 10, "3")
.run([deadline](const Params& q) {
int hop = deadline->run(to_long(q, "budget_ms"), (int)to_long(q, "hops"));
return hop ? "deadline exceeded at hop " + std::to_string(hop)
: std::string("all hops finished in budget");
});
auto slo = std::make_shared<fw::SloBudget>();
registry->add(slo);
p = m.page("SLO error budget", "/operability/slo", "SLO error budget",
"95.0% target over the last 100 events. The error budget is how "
"many bad events the window still allows.",
slo);
p->action("/good", "Good event").run([slo](const Params&) {
slo->record(true);
return "good event recorded";
});
p->action("/bad", "Bad event").run([slo](const Params&) {
slo->record(false);
return "bad event recorded";
});
p->action("/reset", "Reset window").run([slo](const Params&) {
slo->reset();
return "window reset";
});
// ---- operability: chaos, deploy, coordination ---------------------------
auto chaos = std::make_shared<fw::ChaosLab>();
registry->add(chaos);
p = m.page("Chaos experiment", "/operability/chaos", "Chaos experiment",
"Fault injection with a hard blast radius: at most 50 percent and "
"5 faults over 10 calls. Armed 0 is a dry run and injects nothing.",
chaos);
p->action("/run", "Run experiment")
.num("percent", "Percent", 0, fw::labcaps::kMaxPercent, "20")
.num("max_faults", "Max faults", 1, fw::labcaps::kMaxFaults, "2")
.num("armed", "Armed", 0, 1, "0", "0 dry run, 1 armed")
.run([chaos](const Params& q) {
auto r = chaos->run((int)to_long(q, "percent"),
(int)to_long(q, "max_faults"), to_long(q, "armed") == 1);
return std::string(r.armed ? "armed: " : "dry run: ") +
std::to_string(r.injected) + " injected, " +
std::to_string(r.planned) + " planned";
});
auto deploy = std::make_shared<fw::DeployLab>();
registry->add(deploy);
p = m.page("Deploy strategies", "/operability/deploy", "Deploy strategies",
"Rolling replaces one instance per step; blue-green prepares a "
"green fleet and switches in one step. Rollback restores the "
"previous fleet.",
deploy);
p->action("/start", "Start rollout")
.text("strategy", "Strategy", 1, 16, "rolling", "rolling or bluegreen", false)
.text("version", "Version", 1, 16, "v2", "", false)
.run([deploy](const Params& q) {
require_in("strategy", q.at("strategy"), {"rolling", "bluegreen"});
return deploy->start(q.at("strategy"), q.at("version"))
? "rollout started"
: "a rollout is already in progress";
});
p->action("/step", "Next step").run([deploy](const Params&) {
return deploy->step() ? "step done" : "no rollout in progress";
});
p->action("/rollback", "Roll back").run([deploy](const Params&) {
return deploy->rollback() ? "rolled back" : "nothing to roll back";
});
auto cclock = std::make_shared<fw::ManualClock>();
auto coord = std::make_shared<fw::CoordLab>(cclock);
registry->add(coord);
p = m.page("Lease, election, fenced lock", "/operability/coord",
"Coordination",
"Lease, leader election and distributed lock share one rule: a "
"new holder gets a higher fencing token, and writes with a stale "
"token are rejected. Leases last 10 s of injected time.",
coord);
p->action("/acquire", "Acquire")
.text("key", "Key", 1, 16, "lock", "lease, lock or leader", false)
.text("owner", "Owner", 1, 32, "a", "", true)
.run([coord](const Params& q) {
require_in("key", q.at("key"), {"lease", "lock", "leader"});
auto g = coord->acquire(q.at("key"), q.at("owner"));
return g.ok ? "granted token " + std::to_string(g.token)
: "denied, held by " + g.holder;
});
p->action("/release", "Release")
.text("key", "Key", 1, 16, "lock", "lease, lock or leader", false)
.text("owner", "Owner", 1, 32, "a", "", true)
.run([coord](const Params& q) {
require_in("key", q.at("key"), {"lease", "lock", "leader"});
return coord->release(q.at("key"), q.at("owner")) ? "released"
: "not the holder";
});
p->action("/write", "Fenced write")
.text("key", "Key", 1, 16, "lock", "lease, lock or leader", false)
.num("token", "Token", 0, 1000000, "1")
.run([coord](const Params& q) {
require_in("key", q.at("key"), {"lease", "lock", "leader"});
return coord->write(q.at("key"), to_long(q, "token"))
? "write accepted"
: "write rejected: stale fencing token";
});
p->action("/tick", "Advance 6 s").run([coord](const Params&) {
coord->tick(6000);
return "clock advanced 6 s";
});
// ---- supply chain -------------------------------------------------------
auto supply = std::make_shared<fw::SupplyChainLab>();
registry->add(supply);
p = m.page("Supply chain gate", "/supplychain", "Supply chain gate",
"Illustrative SBOM of this example (CycloneDX shape), dependency "
"pins, an advisory scan and an attestation check. The gate passes "
"only when all three pass. The digest is FNV-1a, not a signature.",
supply);
p->action("/advisory", "Publish advisory for chartjs 4.4.1")
.run([supply](const Params&) {
return supply->add_advisory("chartjs", "4.4.1") ? "advisory published"
: "advisory db full";
});
p->action("/tamper", "Tamper with chartjs").run([supply](const Params&) {
supply->tamper();
return "component changed after attestation";
});
p->action("/verify", "Verify attestation")
.text("digest", "Digest", 16, 16, "", "16 hex characters of sbom_digest")
.dynamic_defaults([supply] { return Params{{"digest", supply->attested()}}; })
.run([supply](const Params& q) {
return supply->verify(q.at("digest")) ? "attestation verified"
: "attestation FAILED";
});
p->action("/reset", "Regenerate and attest").run([supply](const Params&) {
supply->reset();
return "sbom regenerated";
});
router.get("/supplychain/sbom.json", "supplychain-sbom",
[supply](fw::Request) {
return fw::JsonResponse(supply->cyclonedx(), 200);
});
// ---- clients depth ------------------------------------------------------
auto retry = std::make_shared<fw::ClientRetryLab>();
registry->add(retry);
p = m.page("Client retry and backoff", "/clients/retry",
"Client retry and backoff",
"In-process transport that fails the first N calls. 5xx is "
"retried with doubling backoff capped at 800 ms and at most 5 "
"attempts; 4xx is never retried. Waits are virtual.",
retry);
p->action("/run", "Send request")
.num("failures", "Failures", 0, 10, "2")
.num("max_attempts", "Max attempts", 1, fw::labcaps::kMaxAttempts, "3")
.num("status", "Failure status", 400, 599, "503")
.run([retry](const Params& q) {
auto o = retry->run((int)to_long(q, "failures"),
(int)to_long(q, "max_attempts"),
(int)to_long(q, "status"));
return "final status " + std::to_string(o.status) + " after " +
std::to_string(o.attempts) + " attempt(s)";
});
auto session = std::make_shared<fw::ClientSessionLab>();
registry->add(session);
p = m.page("Client session: cookies, pool, proxy, auth", "/clients/session",
"Client session",
"Each request goes through the in-process transport: the server "
"sets cookies into the jar, connections are pooled and reused, a "
"proxy change opens a new connection, and the auth scheme sets the "
"Authorization header.",
session);
p->action("/request", "Send request")
.text("auth", "Auth", 1, 8, "none", "none, basic or bearer", false)
.text("proxy", "Proxy", 0, 80, "", "http://host:port or empty", false)
.run([session](const Params& q) {
require_in("auth", q.at("auth"), {"none", "basic", "bearer"});
auto px = q.count("proxy") ? q.at("proxy") : std::string();
if (!fw::ClientSessionLab::valid_proxy(px))
throw fw::ValidationError(
{{"proxy", "must be empty or http(s)/socks5://host:port"}});
auto h = session->request(q.at("auth"), px);
std::string out = "sent:";
for (const auto& [k, v] : h) out += " " + k + "=" + v + ";";
return out;
});
// ---- data depth ---------------------------------------------------------
auto mig = std::make_shared<fw::MigrationLab>();
registry->add(mig);
p = m.page("Migrations: check then apply", "/data/migrations",
"Migrations",
"Preview shows what apply would run without changing anything. "
"Apply runs pending steps in order. Drift drops a table out of "
"band and heal re-creates it from its migration.",
mig);
p->extra_rows([mig] {
fw::LabRows r;
for (const auto& line : mig->check()) r.emplace_back("preview", line);
return r;
});
p->action("/apply", "Apply")
.num("limit", "Steps", 1, 3, "3")
.run([mig](const Params& q) {
int n = mig->apply((int)to_long(q, "limit"));
return std::to_string(n) + " migration(s) applied";
});
p->action("/drift", "Drop posts table").run([mig](const Params&) {
mig->drop_table("posts");
return "posts dropped out of band";
});
p->action("/heal", "Heal drift").run([mig](const Params&) {
return std::to_string(mig->heal()) + " table(s) healed";
});
auto tx = std::make_shared<fw::TxLab>();
registry->add(tx);
p = m.page("Transaction rollback", "/data/transaction", "Transaction rollback",
"Transfer between alice and bob inside a transaction. Overdrafts "
"and a failure after the debit roll back, so no money is lost.",
tx);
p->action("/transfer", "Transfer")
.text("from", "From", 1, 8, "alice", "alice or bob", false)
.text("to", "To", 1, 8, "bob", "alice or bob", false)
.num("amount", "Amount", 1, 1000, "30")
.num("fail_after_debit", "Fail after debit", 0, 1, "0", "0 or 1")
.run([tx](const Params& q) {
require_in("from", q.at("from"), {"alice", "bob"});
require_in("to", q.at("to"), {"alice", "bob"});
if (q.at("from") == q.at("to"))
throw fw::ValidationError({{"to", "must differ from the source"}});
return tx->transfer(q.at("from"), q.at("to"), to_long(q, "amount"),
to_long(q, "fail_after_debit") == 1);
});
auto query = std::make_shared<fw::QueryLab>();
registry->add(query);
p = m.page("Query builder and relations", "/data/query",
"Query builder and relations",
"Builds a joined SELECT over users has-many posts and runs it on "
"in-memory tables. User 0 means all users.",
query);
p->action("/run", "Run query")
.num("user_id", "User id", 0, 2, "0")
.text("order", "Order", 1, 8, "id", "id or title", false)
.num("limit", "Limit", 1, 10, "5")
.run([query](const Params& q) {
require_in("order", q.at("order"), {"id", "title"});
auto r = query->run(to_long(q, "user_id"), q.at("order"),
(int)to_long(q, "limit"));
return std::to_string(r.rows.size()) + " row(s)";
});
auto flight = std::make_shared<fw::SingleFlightLab>();
registry->add(flight);
p = m.page("Singleflight", "/data/singleflight", "Singleflight",
"Concurrent callers asking for the same key share one load. The "
"loader waits for the other callers to join, so the result is "
"deterministic. At most 16 threads.",
flight);
p->action("/run", "Run callers")
.num("callers", "Callers", 1, fw::labcaps::kMaxThreads, "8")
.run([flight](const Params& q) {
int callers = (int)to_long(q, "callers");
int loads = flight->run(callers);
return std::to_string(callers) + " callers, " + std::to_string(loads) +
" load(s)";
});
auto tclock = std::make_shared<fw::ManualClock>();
auto ttl = std::make_shared<fw::TtlLab>(tclock);
registry->add(ttl);
p = m.page("TTL store, LRU, pubsub", "/data/ttl", "TTL store",
"Memory store with per-key TTL on injected time, LRU eviction at "
"5 entries, and a pubsub topic kv.events announcing every change.",
ttl);
p->extra_rows([ttl] {
fw::LabRows r;
for (const auto& e : ttl->events()) r.emplace_back("event", e);
return r;
});
p->action("/set", "Set key")
.text("key", "Key", 1, 16, "", "", true)
.text("value", "Value", 1, 32, "", "", true)
.num("ttl_s", "TTL s", 1, 60, "10")
.run([ttl](const Params& q) {
ttl->set(q.at("key"), q.at("value"), to_long(q, "ttl_s"));
return "stored " + q.at("key");
});
p->action("/get", "Get key")
.text("key", "Key", 1, 16, "", "", true)
.run([ttl](const Params& q) {
std::string v;
return ttl->get(q.at("key"), v) ? "hit: " + v : std::string("miss");
});
p->action("/tick", "Advance 5 s").run([ttl](const Params&) {
ttl->tick(5000);
return "clock advanced 5 s";
});
m.finish();
router.get("/operability/debug", "operability-debug",
[registry](fw::Request) {
return fw::JsonResponse(registry->snapshot(), 200);
});
services.add_feature("Debug snapshot", "/operability/debug");
}
} // namespace molo::example::app