feat: implement MoloApp plans 01-15 with integration
Full C++23 framework tree: runtime, observability, security, data, app, realtime, supervisor, clients, CLI, example plus enterprise tenancy, audit, operability, governance, supplychain. Postgres via docker compose for tests.
This commit is contained in:
@@ -0,0 +1,141 @@
|
||||
# Audit part: required base-tree edits
|
||||
|
||||
The audit part itself is header-only under `src/audit/` and needs no base
|
||||
changes to compile. The five edits below wire it into the framework. Apply in
|
||||
order. Anchors quote the seam API; adjust surrounding context lines to the
|
||||
real files, keep the added lines verbatim.
|
||||
|
||||
Include style: audit headers live in `src/audit/`, so base files reach them
|
||||
by relative path (data layer convention).
|
||||
|
||||
## 1. src/data/orm/field.hpp: PII mark on FieldDef
|
||||
|
||||
Gives every field its compliance mark (see `src/audit/pii/mark.h`). The
|
||||
`Registry::Build` boot check fails the build with the field name when a
|
||||
schema field has no mark.
|
||||
|
||||
```diff
|
||||
--- a/src/data/orm/field.hpp
|
||||
+++ b/src/data/orm/field.hpp
|
||||
@@
|
||||
#pragma once
|
||||
+
|
||||
+#include "../../audit/pii/mark.h"
|
||||
@@
|
||||
struct FieldDef {
|
||||
std::string name;
|
||||
// ... existing members (type, length, nullable, default_value,
|
||||
// has_default, unique, indexed, pk, check) ...
|
||||
+ // PII classification. Default None keeps unclassified columns compiling
|
||||
+ // until Registry::Build boot-fails them by name.
|
||||
+ ::molo::audit::pii::FieldMark pii;
|
||||
};
|
||||
```
|
||||
|
||||
## 2. src/data/orm/model.hpp: ORM write hook
|
||||
|
||||
Calls the audit hook from `Model<Derived>::save` and `Model<Derived>::remove`
|
||||
after the existing `bus->publish` call. `notify_write` no-ops until
|
||||
`Appender::Install` wires a store, so ORM boots without audit configured.
|
||||
|
||||
```diff
|
||||
--- a/src/data/orm/model.hpp
|
||||
+++ b/src/data/orm/model.hpp
|
||||
@@
|
||||
#pragma once
|
||||
+
|
||||
+#include "../../audit/log/appender.h"
|
||||
@@
|
||||
// Model<Derived>::save, after the existing bus->publish(inv) line:
|
||||
if (bus) bus->publish(inv);
|
||||
+ AUDIT(store, table_name(), pk_string(), "save", before_json(), after_json(), ctx, actor);
|
||||
@@
|
||||
// Model<Derived>::remove, after the existing bus->publish(inv) line:
|
||||
if (bus) bus->publish(inv);
|
||||
+ AUDIT(store, table_name(), pk_string(), "delete", before_json(), "", ctx, actor);
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- `AUDIT` is defined in `src/audit/log/appender.h` and forwards to
|
||||
`::molo::audit::log::notify_write(table, pk, op, before, after, ctx, actor)`.
|
||||
- `table_name()`, `pk_string()`, `before_json()`, `after_json()` are the
|
||||
call-site spellings for the existing `table_name`/`primary_key` statics and
|
||||
the `get`/`values` accessors; use whatever locals the real `save`/`remove`
|
||||
already hold.
|
||||
- `actor` is the ambient subject string when the call site has one, else `""`.
|
||||
|
||||
## 3. src/observability/logging/registry.h + registry.cpp: redact on emit
|
||||
|
||||
Passes every record through the redactor before fan-out to sinks, so PII can
|
||||
never reach a log file through any logger.
|
||||
|
||||
```diff
|
||||
--- a/src/observability/logging/registry.h
|
||||
+++ b/src/observability/logging/registry.h
|
||||
@@
|
||||
#pragma once
|
||||
+
|
||||
+#include "../../audit/redact/redactor.h"
|
||||
@@
|
||||
void emit(Record r) {
|
||||
+ ::molo::audit::redact::Redactor::RedactRecord(r);
|
||||
if (hook_) hook_(r);
|
||||
for (auto& s : sinks_) s->write(r);
|
||||
}
|
||||
```
|
||||
|
||||
If the real `emit` lives in `registry.cpp` instead of the header, put the
|
||||
include and the added line there.
|
||||
|
||||
## 4. src/app/errors/handler.h: redact error detail
|
||||
|
||||
Runs user-visible detail and internal diagnostics through the redactor before
|
||||
they reach the response body and the log sink.
|
||||
|
||||
```diff
|
||||
--- a/src/app/errors/handler.h
|
||||
+++ b/src/app/errors/handler.h
|
||||
@@
|
||||
#pragma once
|
||||
+
|
||||
+#include "../../audit/redact/redactor.h"
|
||||
@@
|
||||
Response render(const HttpError& e, const Request& req, const std::string& request_id) {
|
||||
+ const std::string safe_detail =
|
||||
+ ::molo::audit::redact::Redactor::RedactDetail(e.detail());
|
||||
// ... build body from safe_detail instead of e.detail() ...
|
||||
}
|
||||
@@
|
||||
Response render_unknown(const std::string& internal, const std::string& request_id) {
|
||||
+ const std::string safe_internal =
|
||||
+ ::molo::audit::redact::Redactor::RedactDetail(internal);
|
||||
// ... sink + body use safe_internal ...
|
||||
}
|
||||
```
|
||||
|
||||
## 5. src/security/auth/Permission.h: audit grants
|
||||
|
||||
Declares the three permission names the audit query path enforces via
|
||||
`Guard::check` (`audit.read` today; `audit.query` and `rights.admin` reserved
|
||||
for the query service and the rights-request console).
|
||||
|
||||
```diff
|
||||
--- a/src/security/auth/Permission.h
|
||||
+++ b/src/security/auth/Permission.h
|
||||
@@
|
||||
namespace molo::security::auth {
|
||||
+
|
||||
+inline const Permission kPermAuditRead{"audit.read"};
|
||||
+inline const Permission kPermAuditQuery{"audit.query"};
|
||||
+inline const Permission kPermRightsAdmin{"rights.admin"};
|
||||
+
|
||||
// ... existing permission declarations ...
|
||||
}
|
||||
```
|
||||
|
||||
No `Application` edit is needed: audit middleware and routes register at
|
||||
deploy composition, and `handle()` already sets the request id pre-chain.
|
||||
Blocking audit work (`Enforcer::Apply`, `Exporter::Build`, `Eraser::Erase`,
|
||||
`Fixture::Scrub`) stays synchronous for the supervisor scheduler and is never
|
||||
called from request handlers.
|
||||
@@ -0,0 +1,670 @@
|
||||
// ADAPTER: minimal local stand-ins for base-tree seam headers that are
|
||||
// absent in an isolated checkout. Every block below prefers the real header
|
||||
// via __has_include and only defines a stub when it is missing. Stubs carry
|
||||
// the exact seam names so the audit code compiles unchanged against the real
|
||||
// tree. Delete nothing here when integrating: the guards go quiet on their
|
||||
// own once the base tree is present.
|
||||
#pragma once
|
||||
|
||||
#include <array>
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <source_location>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <utility>
|
||||
#include <variant>
|
||||
#include <vector>
|
||||
|
||||
// ---------------------------------------------------------------- crypto
|
||||
#if __has_include("security/crypto/Sha256.h")
|
||||
#include "security/crypto/Sha256.h"
|
||||
#elif __has_include("src/security/crypto/Sha256.h")
|
||||
#include "src/security/crypto/Sha256.h"
|
||||
#else
|
||||
// ADAPTER: compact real SHA-256 (FIPS 180-4) so chain/seal code and tests run
|
||||
// standalone. Interface mirrors the seam: Sha256::hash/hex/update/final.
|
||||
namespace molo::security::crypto {
|
||||
class Sha256 {
|
||||
public:
|
||||
static std::array<unsigned char, 32> hash(std::string_view msg) {
|
||||
Sha256 h;
|
||||
h.update(msg);
|
||||
return h.final();
|
||||
}
|
||||
static std::string hex(const std::array<unsigned char, 32>& d) {
|
||||
static constexpr char kHex[] = "0123456789abcdef";
|
||||
std::string out;
|
||||
out.resize(64);
|
||||
for (int i = 0; i < 32; ++i) {
|
||||
out[2 * i] = kHex[(d[i] >> 4) & 15];
|
||||
out[2 * i + 1] = kHex[d[i] & 15];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
void update(std::string_view msg) {
|
||||
for (char c : msg) {
|
||||
block_[used_++] = static_cast<unsigned char>(c);
|
||||
++total_;
|
||||
if (used_ == 64) {
|
||||
compress();
|
||||
used_ = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
std::array<unsigned char, 32> final() {
|
||||
std::uint64_t bits = total_ * 8;
|
||||
unsigned char pad = 0x80;
|
||||
update({reinterpret_cast<const char*>(&pad), 1});
|
||||
unsigned char zero = 0;
|
||||
while (used_ != 56) update({reinterpret_cast<const char*>(&zero), 1});
|
||||
for (int i = 7; i >= 0; --i)
|
||||
update({reinterpret_cast<const char*>(&kZero_) + 0, 0}), feed_byte((bits >> (i * 8)) & 0xff);
|
||||
std::array<unsigned char, 32> d{};
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
d[4 * i] = (s_[i] >> 24) & 0xff;
|
||||
d[4 * i + 1] = (s_[i] >> 16) & 0xff;
|
||||
d[4 * i + 2] = (s_[i] >> 8) & 0xff;
|
||||
d[4 * i + 3] = s_[i] & 0xff;
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
private:
|
||||
void feed_byte(unsigned char b) {
|
||||
block_[used_++] = b;
|
||||
if (used_ == 64) {
|
||||
compress();
|
||||
used_ = 0;
|
||||
}
|
||||
}
|
||||
static std::uint32_t rotr(std::uint32_t x, int n) { return (x >> n) | (x << (32 - n)); }
|
||||
void compress() {
|
||||
static constexpr std::uint32_t k[64] = {
|
||||
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4,
|
||||
0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe,
|
||||
0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f,
|
||||
0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
|
||||
0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc,
|
||||
0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b,
|
||||
0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116,
|
||||
0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
|
||||
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7,
|
||||
0xc67178f2};
|
||||
std::uint32_t w[64];
|
||||
for (int i = 0; i < 16; ++i)
|
||||
w[i] = (std::uint32_t(block_[4 * i]) << 24) | (std::uint32_t(block_[4 * i + 1]) << 16) |
|
||||
(std::uint32_t(block_[4 * i + 2]) << 8) | std::uint32_t(block_[4 * i + 3]);
|
||||
for (int i = 16; i < 64; ++i) {
|
||||
std::uint32_t s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3);
|
||||
std::uint32_t s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10);
|
||||
w[i] = w[i - 16] + s0 + w[i - 7] + s1;
|
||||
}
|
||||
std::uint32_t v[8];
|
||||
for (int i = 0; i < 8; ++i) v[i] = s_[i];
|
||||
for (int i = 0; i < 64; ++i) {
|
||||
std::uint32_t s1 = rotr(v[4], 6) ^ rotr(v[4], 11) ^ rotr(v[4], 25);
|
||||
std::uint32_t ch = (v[4] & v[5]) ^ (~v[4] & v[6]);
|
||||
std::uint32_t t1 = v[7] + s1 + ch + k[i] + w[i];
|
||||
std::uint32_t s0 = rotr(v[0], 2) ^ rotr(v[0], 13) ^ rotr(v[0], 22);
|
||||
std::uint32_t mj = (v[0] & v[1]) ^ (v[0] & v[2]) ^ (v[1] & v[2]);
|
||||
std::uint32_t t2 = s0 + mj;
|
||||
v[7] = v[6];
|
||||
v[6] = v[5];
|
||||
v[5] = v[4];
|
||||
v[4] = v[3] + t1;
|
||||
v[3] = v[2];
|
||||
v[2] = v[1];
|
||||
v[1] = v[0];
|
||||
v[0] = t1 + t2;
|
||||
}
|
||||
for (int i = 0; i < 8; ++i) s_[i] += v[i];
|
||||
}
|
||||
std::uint32_t s_[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a,
|
||||
0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19};
|
||||
unsigned char block_[64] = {};
|
||||
std::size_t used_ = 0;
|
||||
std::uint64_t total_ = 0;
|
||||
static constexpr char kZero_ = 0;
|
||||
};
|
||||
} // namespace molo::security::crypto
|
||||
#endif
|
||||
|
||||
#if __has_include("security/crypto/Hmac.h")
|
||||
#include "security/crypto/Hmac.h"
|
||||
#elif __has_include("src/security/crypto/Hmac.h")
|
||||
#include "src/security/crypto/Hmac.h"
|
||||
#else
|
||||
// ADAPTER: HMAC-SHA256 over the adapter Sha256 above.
|
||||
namespace molo::security::crypto {
|
||||
class Hmac {
|
||||
public:
|
||||
static std::array<unsigned char, 32> sha256(std::string_view key, std::string_view msg) {
|
||||
unsigned char kb[64] = {};
|
||||
if (key.size() > 64) {
|
||||
auto h = Sha256::hash(key);
|
||||
for (int i = 0; i < 32; ++i) kb[i] = h[i];
|
||||
} else {
|
||||
for (std::size_t i = 0; i < key.size(); ++i) kb[i] = static_cast<unsigned char>(key[i]);
|
||||
}
|
||||
std::string ik, ok;
|
||||
for (int i = 0; i < 64; ++i) {
|
||||
ik.push_back(static_cast<char>(kb[i] ^ 0x36));
|
||||
ok.push_back(static_cast<char>(kb[i] ^ 0x5c));
|
||||
}
|
||||
Sha256 inner;
|
||||
inner.update(ik);
|
||||
inner.update(msg);
|
||||
auto ih = inner.final();
|
||||
Sha256 outer;
|
||||
outer.update(ok);
|
||||
outer.update({reinterpret_cast<const char*>(ih.data()), 32});
|
||||
return outer.final();
|
||||
}
|
||||
static std::string hex(const std::array<unsigned char, 32>& d) { return Sha256::hex(d); }
|
||||
};
|
||||
} // namespace molo::security::crypto
|
||||
#endif
|
||||
|
||||
// ---------------------------------------------------------------- data value
|
||||
#if __has_include("data/value.hpp")
|
||||
#include "data/value.hpp"
|
||||
#elif __has_include("src/data/value.hpp")
|
||||
#include "src/data/value.hpp"
|
||||
#else
|
||||
// ADAPTER: seam-compatible Value variant + makers.
|
||||
namespace molo::data {
|
||||
using Bytes = std::vector<unsigned char>;
|
||||
using TimePoint = std::chrono::system_clock::time_point;
|
||||
using Value = std::variant<std::monostate, std::int64_t, double, std::string, Bytes, bool, TimePoint>;
|
||||
inline Value make_null() { return Value{std::monostate{}}; }
|
||||
inline Value make_int(std::int64_t v) { return Value{v}; }
|
||||
inline Value make_double(double v) { return Value{v}; }
|
||||
inline Value make_text(std::string v) { return Value{std::move(v)}; }
|
||||
inline Value make_bytes(Bytes v) { return Value{std::move(v)}; }
|
||||
inline Value make_bool(bool v) { return Value{v}; }
|
||||
inline Value make_time(TimePoint v) { return Value{v}; }
|
||||
} // namespace molo::data
|
||||
#endif
|
||||
|
||||
// ---------------------------------------------------------------- data error
|
||||
#if __has_include("data/error.hpp")
|
||||
#include "data/error.hpp"
|
||||
#elif __has_include("src/data/error.hpp")
|
||||
#include "src/data/error.hpp"
|
||||
#else
|
||||
// ADAPTER: seam-compatible DbError.
|
||||
namespace molo::data {
|
||||
enum class Backend { kPostgres, kSqlite };
|
||||
class DbError {
|
||||
public:
|
||||
DbError(Backend b, std::string c, std::string m, std::string n = "")
|
||||
: backend_(b), code_(std::move(c)), msg_(std::move(m)), native_(std::move(n)) {}
|
||||
Backend backend() const { return backend_; }
|
||||
const std::string& code() const { return code_; }
|
||||
const std::string& detail() const { return msg_; }
|
||||
bool healable() const { return false; }
|
||||
bool retryable_txn() const { return false; }
|
||||
|
||||
private:
|
||||
Backend backend_;
|
||||
std::string code_, msg_, native_;
|
||||
};
|
||||
} // namespace molo::data
|
||||
#endif
|
||||
|
||||
// ---------------------------------------------------------------- data context
|
||||
#if __has_include("data/context.hpp")
|
||||
#include "data/context.hpp"
|
||||
#elif __has_include("src/data/context.hpp")
|
||||
#include "src/data/context.hpp"
|
||||
#else
|
||||
// ADAPTER: seam-compatible Context.
|
||||
namespace molo::data {
|
||||
class Context {
|
||||
public:
|
||||
Context(std::string cid, std::string tenant, int timeout_sec = 30)
|
||||
: cid_(std::move(cid)), tenant_(std::move(tenant)), timeout_(timeout_sec) {}
|
||||
static Context background(std::string why) { return Context("bg:" + why, "", 30); }
|
||||
const std::string& correlation_id() const { return cid_; }
|
||||
const std::string& tenant() const { return tenant_; }
|
||||
bool expired() const { return false; }
|
||||
|
||||
private:
|
||||
std::string cid_, tenant_;
|
||||
int timeout_;
|
||||
};
|
||||
} // namespace molo::data
|
||||
#endif
|
||||
|
||||
// ---------------------------------------------------------------- data statement
|
||||
#if __has_include("data/statement.hpp")
|
||||
#include "data/statement.hpp"
|
||||
#elif __has_include("src/data/statement.hpp")
|
||||
#include "src/data/statement.hpp"
|
||||
#else
|
||||
// ADAPTER: seam-compatible BoundStatement + check_identifier.
|
||||
namespace molo::data {
|
||||
inline void check_identifier(const std::string& s) {
|
||||
for (char c : s)
|
||||
if (!(c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')))
|
||||
throw DbError(Backend::kSqlite, "bad_identifier", s);
|
||||
if (s.empty() || s.size() > 128) throw DbError(Backend::kSqlite, "bad_identifier", s);
|
||||
}
|
||||
class BoundStatement {
|
||||
public:
|
||||
BoundStatement(std::string sql, std::vector<Value> params)
|
||||
: sql_(std::move(sql)), params_(std::move(params)) {}
|
||||
static BoundStatement raw(std::string sql, std::vector<Value> params = {}) {
|
||||
return BoundStatement(std::move(sql), std::move(params));
|
||||
}
|
||||
const std::string& sql() const { return sql_; }
|
||||
const std::vector<Value>& params() const { return params_; }
|
||||
|
||||
private:
|
||||
std::string sql_;
|
||||
std::vector<Value> params_;
|
||||
};
|
||||
} // namespace molo::data
|
||||
#endif
|
||||
|
||||
// ---------------------------------------------------------------- data connection
|
||||
#if __has_include("data/connection.hpp")
|
||||
#include "data/connection.hpp"
|
||||
#elif __has_include("src/data/connection.hpp")
|
||||
#include "src/data/connection.hpp"
|
||||
#else
|
||||
// ADAPTER: in-memory Connection stub with a tiny SQL surface (CREATE TABLE,
|
||||
// INSERT, SELECT with WHERE seq range, DELETE with WHERE) sufficient for the
|
||||
// audit SqlStore standalone. Not a database: unknown statements fail loudly.
|
||||
namespace molo::data {
|
||||
class Connection {
|
||||
public:
|
||||
using Row = std::vector<Value>;
|
||||
struct Result {
|
||||
std::vector<Row> rows;
|
||||
std::uint64_t affected = 0;
|
||||
};
|
||||
explicit Connection(Backend b = Backend::kSqlite) : backend_(b) {}
|
||||
Backend backend() const { return backend_; }
|
||||
std::string backend_id() const { return backend_ == Backend::kPostgres ? "pg" : "sqlite"; }
|
||||
bool is_postgres() const { return backend_ == Backend::kPostgres; }
|
||||
// ADAPTER has no SQL engine: audit SqlStore binds through SqlExec
|
||||
// instead (see log/sql_store.h). Any direct call fails loudly.
|
||||
Result exec(const Context&, const BoundStatement&) {
|
||||
throw DbError(backend_, "adapter_no_sql", "adapter Connection has no SQL engine");
|
||||
}
|
||||
|
||||
private:
|
||||
Backend backend_;
|
||||
std::map<std::string, std::vector<Row>> tables_;
|
||||
};
|
||||
} // namespace molo::data
|
||||
#endif
|
||||
|
||||
// ---------------------------------------------------------------- kv store
|
||||
#if __has_include("data/kv/store.hpp")
|
||||
#include "data/kv/store.hpp"
|
||||
#elif __has_include("src/data/kv/store.hpp")
|
||||
#include "src/data/kv/store.hpp"
|
||||
#else
|
||||
// ADAPTER: in-memory KvStore with the seam surface.
|
||||
namespace molo::data::kv {
|
||||
struct ScanPage {
|
||||
std::vector<std::pair<std::string, Bytes>> items;
|
||||
std::string next_cursor;
|
||||
bool done = true;
|
||||
};
|
||||
class KvStore {
|
||||
public:
|
||||
std::optional<Bytes> get(const std::string& k) {
|
||||
auto it = m_.find(k);
|
||||
if (it == m_.end()) return std::nullopt;
|
||||
return it->second;
|
||||
}
|
||||
void set(std::string k, Bytes v, int = 0) { m_[std::move(k)] = std::move(v); }
|
||||
bool del(const std::string& k) { return m_.erase(k) > 0; }
|
||||
bool exists(const std::string& k) { return m_.count(k) > 0; }
|
||||
ScanPage scan(const std::string& prefix, const std::string& cursor, std::size_t limit) {
|
||||
ScanPage p;
|
||||
bool started = cursor.empty();
|
||||
for (auto& [k, v] : m_) {
|
||||
if (k.rfind(prefix, 0) != 0) continue;
|
||||
if (!started) {
|
||||
if (k == cursor) started = true;
|
||||
continue;
|
||||
}
|
||||
p.items.emplace_back(k, v);
|
||||
if (p.items.size() >= limit) {
|
||||
p.done = false;
|
||||
p.next_cursor = k;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
private:
|
||||
std::map<std::string, Bytes> m_;
|
||||
};
|
||||
} // namespace molo::data::kv
|
||||
#endif
|
||||
|
||||
// ---------------------------------------------------------------- cache
|
||||
#if __has_include("data/cache/cache.hpp")
|
||||
#include "data/cache/cache.hpp"
|
||||
#elif __has_include("src/data/cache/cache.hpp")
|
||||
#include "src/data/cache/cache.hpp"
|
||||
#else
|
||||
// ADAPTER: in-memory Cache with the seam surface.
|
||||
namespace molo::data::cache {
|
||||
inline std::string tag_for_table(const std::string& t) { return "table:" + t; }
|
||||
class Cache {
|
||||
public:
|
||||
std::optional<Bytes> get(const std::string& k) {
|
||||
auto it = m_.find(k);
|
||||
if (it == m_.end()) return std::nullopt;
|
||||
return it->second;
|
||||
}
|
||||
void put(std::string k, Bytes v, std::vector<std::string> tags = {}, int = 0) {
|
||||
for (auto& t : tags) tags_[t].push_back(k);
|
||||
m_[std::move(k)] = std::move(v);
|
||||
}
|
||||
bool invalidate(const std::string& k) { return m_.erase(k) > 0; }
|
||||
std::int64_t purge_tag(const std::string& t) {
|
||||
std::int64_t n = 0;
|
||||
auto it = tags_.find(t);
|
||||
if (it == tags_.end()) return 0;
|
||||
for (auto& k : it->second) n += m_.erase(k);
|
||||
tags_.erase(it);
|
||||
return n;
|
||||
}
|
||||
|
||||
private:
|
||||
std::map<std::string, Bytes> m_;
|
||||
std::map<std::string, std::vector<std::string>> tags_;
|
||||
};
|
||||
} // namespace molo::data::cache
|
||||
#endif
|
||||
|
||||
// ---------------------------------------------------------------- invalidation bus
|
||||
#if __has_include("data/orm/invalidation.hpp")
|
||||
#include "data/orm/invalidation.hpp"
|
||||
#elif __has_include("src/data/orm/invalidation.hpp")
|
||||
#include "src/data/orm/invalidation.hpp"
|
||||
#else
|
||||
// ADAPTER: seam-compatible Invalidation + InvalidationBus.
|
||||
namespace molo::data::orm {
|
||||
struct Invalidation {
|
||||
std::string table, pk, op;
|
||||
std::int64_t txid = 0;
|
||||
};
|
||||
class InvalidationBus {
|
||||
public:
|
||||
using Handler = std::function<void(const Invalidation&)>;
|
||||
int subscribe(Handler h) {
|
||||
handlers_[next_] = std::move(h);
|
||||
return next_++;
|
||||
}
|
||||
void unsubscribe(int id) { handlers_.erase(id); }
|
||||
void publish(const Invalidation& iv) {
|
||||
for (auto& [_, h] : handlers_) h(iv);
|
||||
}
|
||||
|
||||
private:
|
||||
std::map<int, Handler> handlers_;
|
||||
int next_ = 1;
|
||||
};
|
||||
} // namespace molo::data::orm
|
||||
#endif
|
||||
|
||||
// ---------------------------------------------------------------- metric sink
|
||||
#if __has_include("data/obs.hpp")
|
||||
#include "data/obs.hpp"
|
||||
#elif __has_include("src/data/obs.hpp")
|
||||
#include "src/data/obs.hpp"
|
||||
#else
|
||||
// ADAPTER: seam-compatible MetricSink + null_sink.
|
||||
namespace molo::data::obs {
|
||||
class MetricSink {
|
||||
public:
|
||||
virtual ~MetricSink() = default;
|
||||
virtual void counter(const std::string& n, double d,
|
||||
const std::vector<std::pair<std::string, std::string>>& l = {}) = 0;
|
||||
virtual void gauge(const std::string& n, double v,
|
||||
const std::vector<std::pair<std::string, std::string>>& l = {}) = 0;
|
||||
virtual void histogram(const std::string& n, double v,
|
||||
const std::vector<std::pair<std::string, std::string>>& l = {}) = 0;
|
||||
};
|
||||
class NullSink : public MetricSink {
|
||||
public:
|
||||
void counter(const std::string&, double,
|
||||
const std::vector<std::pair<std::string, std::string>>&) override {}
|
||||
void gauge(const std::string&, double,
|
||||
const std::vector<std::pair<std::string, std::string>>&) override {}
|
||||
void histogram(const std::string&, double,
|
||||
const std::vector<std::pair<std::string, std::string>>&) override {}
|
||||
};
|
||||
inline std::shared_ptr<MetricSink> null_sink() { return std::make_shared<NullSink>(); }
|
||||
} // namespace molo::data::obs
|
||||
#endif
|
||||
|
||||
// ---------------------------------------------------------------- log record/sink
|
||||
#if __has_include("observability/logging/record.h")
|
||||
#include "observability/logging/record.h"
|
||||
#elif __has_include("src/observability/logging/record.h")
|
||||
#include "src/observability/logging/record.h"
|
||||
#else
|
||||
// ADAPTER: seam-compatible log Record/Level/Field.
|
||||
namespace molo::log {
|
||||
enum class Level { kTrace, kDebug, kInfo, kWarn, kError, kFatal };
|
||||
struct Field {
|
||||
std::string key, value;
|
||||
};
|
||||
struct Record {
|
||||
std::int64_t timestamp = 0;
|
||||
Level level = Level::kInfo;
|
||||
std::string logger, message, file, func, request_id, trace_id, span_id;
|
||||
int line = 0;
|
||||
std::vector<Field> fields;
|
||||
bool truncated = false;
|
||||
};
|
||||
} // namespace molo::log
|
||||
#endif
|
||||
|
||||
#if __has_include("observability/logging/sink.h")
|
||||
#include "observability/logging/sink.h"
|
||||
#elif __has_include("src/observability/logging/sink.h")
|
||||
#include "src/observability/logging/sink.h"
|
||||
#else
|
||||
// ADAPTER: seam-compatible Sink.
|
||||
namespace molo::log {
|
||||
class Sink {
|
||||
public:
|
||||
virtual ~Sink() = default;
|
||||
virtual void write(const Record& r) = 0;
|
||||
virtual void flush() {}
|
||||
};
|
||||
} // namespace molo::log
|
||||
#endif
|
||||
|
||||
#if __has_include("observability/logging/registry.h")
|
||||
#include "observability/logging/registry.h"
|
||||
#elif __has_include("src/observability/logging/registry.h")
|
||||
#include "src/observability/logging/registry.h"
|
||||
#else
|
||||
// ADAPTER: seam-compatible Registry (emit path only).
|
||||
namespace molo::log {
|
||||
class Registry {
|
||||
public:
|
||||
static Registry& global() {
|
||||
static Registry r;
|
||||
return r;
|
||||
}
|
||||
void add_sink(std::shared_ptr<Sink> s) { sinks_.push_back(std::move(s)); }
|
||||
void emit(Record r) {
|
||||
if (hook_) hook_(r);
|
||||
for (auto& s : sinks_) s->write(r);
|
||||
}
|
||||
void set_record_hook(std::function<void(Record&)> h) { hook_ = std::move(h); }
|
||||
|
||||
private:
|
||||
std::vector<std::shared_ptr<Sink>> sinks_;
|
||||
std::function<void(Record&)> hook_;
|
||||
};
|
||||
} // namespace molo::log
|
||||
#endif
|
||||
|
||||
#if __has_include("observability/logging/logger.h")
|
||||
#include "observability/logging/logger.h"
|
||||
#elif __has_include("src/observability/logging/logger.h")
|
||||
#include "src/observability/logging/logger.h"
|
||||
#else
|
||||
// ADAPTER: seam-compatible Logger surface used by audit.
|
||||
namespace molo::log {
|
||||
class Logger {
|
||||
public:
|
||||
explicit Logger(std::string n) : name_(std::move(n)) {}
|
||||
void log(Level lv, std::string_view msg, std::source_location loc = std::source_location::current(),
|
||||
std::vector<Field> f = {}) {
|
||||
Record r;
|
||||
r.level = lv;
|
||||
r.logger = name_;
|
||||
r.message = std::string(msg);
|
||||
r.file = loc.file_name();
|
||||
r.line = static_cast<int>(loc.line());
|
||||
r.func = loc.function_name();
|
||||
r.fields = std::move(f);
|
||||
Registry::global().emit(std::move(r));
|
||||
}
|
||||
|
||||
private:
|
||||
std::string name_;
|
||||
};
|
||||
} // namespace molo::log
|
||||
#endif
|
||||
|
||||
// ---------------------------------------------------------------- correlation
|
||||
#if __has_include("observability/correlation/request_context.h")
|
||||
#include "observability/correlation/request_context.h"
|
||||
#elif __has_include("src/observability/correlation/request_context.h")
|
||||
#include "src/observability/correlation/request_context.h"
|
||||
#else
|
||||
// ADAPTER: seam-compatible RequestContext.
|
||||
namespace molo::corr {
|
||||
class RequestContext {
|
||||
public:
|
||||
explicit RequestContext(std::string rid) : rid_(std::move(rid)) {}
|
||||
const std::string& request_id() const { return rid_; }
|
||||
|
||||
private:
|
||||
std::string rid_;
|
||||
};
|
||||
} // namespace molo::corr
|
||||
#endif
|
||||
|
||||
#if __has_include("observability/correlation/redaction.h")
|
||||
#include "observability/correlation/redaction.h"
|
||||
#elif __has_include("src/observability/correlation/redaction.h")
|
||||
#include "src/observability/correlation/redaction.h"
|
||||
#else
|
||||
// ADAPTER: seam-compatible sensitivity helpers.
|
||||
namespace molo::corr {
|
||||
inline bool is_sensitive_key(std::string_view key) {
|
||||
static constexpr const char* k[] = {"password", "passwd", "secret", "token", "api_key",
|
||||
"apikey", "auth", "ssn", "card", "iban", "credential"};
|
||||
for (auto* s : k)
|
||||
if (key.find(s) != std::string_view::npos) return true;
|
||||
return false;
|
||||
}
|
||||
inline std::string redact_value(std::string_view, std::string_view) { return "[redacted]"; }
|
||||
} // namespace molo::corr
|
||||
#endif
|
||||
|
||||
// ---------------------------------------------------------------- metrics counter
|
||||
#if __has_include("observability/metrics/counter.h")
|
||||
#include "observability/metrics/counter.h"
|
||||
#elif __has_include("src/observability/metrics/counter.h")
|
||||
#include "src/observability/metrics/counter.h"
|
||||
#else
|
||||
// ADAPTER: seam-compatible Counter.
|
||||
namespace molo::metrics {
|
||||
class Counter {
|
||||
public:
|
||||
Counter(std::string n, std::string h, std::vector<std::string> l = {})
|
||||
: name_(std::move(n)), help_(std::move(h)), labels_(std::move(l)) {}
|
||||
void inc(double d = 1.0) { value_ += d; }
|
||||
double value() const { return value_; }
|
||||
|
||||
private:
|
||||
std::string name_, help_;
|
||||
std::vector<std::string> labels_;
|
||||
double value_ = 0;
|
||||
};
|
||||
} // namespace molo::metrics
|
||||
#endif
|
||||
|
||||
// ---------------------------------------------------------------- http request
|
||||
#if __has_include("app/http_msg/request.h")
|
||||
#include "app/http_msg/request.h"
|
||||
#elif __has_include("src/app/http_msg/request.h")
|
||||
#include "src/app/http_msg/request.h"
|
||||
#else
|
||||
// ADAPTER: seam-compatible Request surface used by audit.
|
||||
namespace molo::app::http_msg {
|
||||
class Request {
|
||||
public:
|
||||
const std::string& request_id() const { return rid_; }
|
||||
void set_request_id(std::string v) { rid_ = std::move(v); }
|
||||
std::string route() const { return route_; }
|
||||
|
||||
private:
|
||||
std::string rid_, route_;
|
||||
};
|
||||
} // namespace molo::app::http_msg
|
||||
#endif
|
||||
|
||||
// ---------------------------------------------------------------- auth guard
|
||||
#if __has_include("security/auth/Guard.h")
|
||||
#include "security/auth/Guard.h"
|
||||
#elif __has_include("src/security/auth/Guard.h")
|
||||
#include "src/security/auth/Guard.h"
|
||||
#else
|
||||
// ADAPTER: seam-compatible Guard/Permission/Role/Session/Verdict.
|
||||
namespace molo::security::auth {
|
||||
struct Permission {
|
||||
std::string name;
|
||||
};
|
||||
struct Role {
|
||||
std::string name;
|
||||
std::vector<std::string> grants;
|
||||
};
|
||||
struct Session {
|
||||
std::string subject;
|
||||
std::string tenant;
|
||||
std::vector<Role> roles;
|
||||
};
|
||||
struct Verdict {
|
||||
bool allowed = false;
|
||||
int http_status = 403;
|
||||
std::string code = "forbidden";
|
||||
};
|
||||
class Guard {
|
||||
public:
|
||||
static Verdict check(const Session& s, const std::vector<Role>&, const std::vector<Permission>& req) {
|
||||
for (auto& p : req) {
|
||||
bool ok = false;
|
||||
for (auto& r : s.roles)
|
||||
for (auto& g : r.grants)
|
||||
if (g == p.name) ok = true;
|
||||
if (!ok) return Verdict{false, 403, "forbidden"};
|
||||
}
|
||||
return Verdict{true, 200, "ok"};
|
||||
}
|
||||
};
|
||||
} // namespace molo::security::auth
|
||||
#endif
|
||||
@@ -0,0 +1,24 @@
|
||||
// MoloApp audit trail and compliance part: single umbrella include.
|
||||
#pragma once
|
||||
|
||||
#include "adapt.h"
|
||||
#include "caps.h"
|
||||
#include "log/appender.h"
|
||||
#include "log/chain.h"
|
||||
#include "log/query.h"
|
||||
#include "log/record.h"
|
||||
#include "log/sql_store.h"
|
||||
#include "log/store.h"
|
||||
#include "mask/fixture.h"
|
||||
#include "mask/masker.h"
|
||||
#include "pii/mark.h"
|
||||
#include "pii/registry.h"
|
||||
#include "redact/pattern.h"
|
||||
#include "redact/redactor.h"
|
||||
#include "retention/enforcer.h"
|
||||
#include "retention/hold.h"
|
||||
#include "retention/policy.h"
|
||||
#include "rights/eraser.h"
|
||||
#include "rights/exporter.h"
|
||||
#include "rights/request.h"
|
||||
#include "rights/subject.h"
|
||||
@@ -0,0 +1,40 @@
|
||||
// MoloApp audit trail and compliance part: static capacity ceilings.
|
||||
// Every knob in this part is a compile-time constant. No autotuning.
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
|
||||
namespace molo::audit {
|
||||
|
||||
// Single audit record after JSON encoding, including before/after images.
|
||||
inline constexpr std::size_t kMaxRecordBytes = 64 * 1024;
|
||||
// Before/after image payloads inside one record.
|
||||
inline constexpr std::size_t kMaxBeforeBytes = 16 * 1024;
|
||||
inline constexpr std::size_t kMaxAfterBytes = 16 * 1024;
|
||||
// Chain seal cadence: every N records or every T seconds, whichever first.
|
||||
inline constexpr std::uint64_t kSealEveryRecords = 1000;
|
||||
inline constexpr std::uint64_t kSealEverySeconds = 60;
|
||||
// Max rows returned by one audit query page.
|
||||
inline constexpr std::size_t kQueryPageMax = 1000;
|
||||
// Subject export bundle limits.
|
||||
inline constexpr std::size_t kExportMaxBytes = 256ULL * 1024 * 1024;
|
||||
inline constexpr std::size_t kExportMaxRows = 1000000;
|
||||
// Max subject rows erased by one erasure run.
|
||||
inline constexpr std::size_t kErasureMaxPerRun = 100000;
|
||||
// Max rows scanned by one retention enforcement run.
|
||||
inline constexpr std::size_t kRetentionScanMaxPerRun = 1000000;
|
||||
// Max input bytes redacted by one Redactor call.
|
||||
inline constexpr std::size_t kRedactMaxBytesPerCall = 1024 * 1024;
|
||||
// Max live retention holds process-wide.
|
||||
inline constexpr std::size_t kMaxHolds = 100000;
|
||||
// Max open rights requests per tenant.
|
||||
inline constexpr std::size_t kMaxOpenRequestsPerTenant = 10000;
|
||||
// Fixture scrub chunk size (1MB streaming, never full buffering).
|
||||
inline constexpr std::size_t kFixtureChunkBytes = 1024 * 1024;
|
||||
// Max audit fields per log record redaction pass.
|
||||
inline constexpr std::size_t kRedactMaxFields = 256;
|
||||
// Max PII hits processed by one redaction/scrub call.
|
||||
inline constexpr std::size_t kRedactMaxHitsPerCall = 65536;
|
||||
|
||||
} // namespace molo::audit
|
||||
@@ -0,0 +1,124 @@
|
||||
// Appender: the write path for audit records. ORM save()/remove() reach it
|
||||
// through notify_write (see PATCHES.md entry 2); the AUDIT macro fills
|
||||
// actor, subject, request and trace ids from the ambient call context.
|
||||
// Oversize before/after images are truncated to their caps; oversize records
|
||||
// are rejected loudly, never stored half.
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
|
||||
#include "../adapt.h"
|
||||
#include "../caps.h"
|
||||
#include "record.h"
|
||||
#include "store.h"
|
||||
|
||||
namespace molo::audit::log {
|
||||
|
||||
class Appender {
|
||||
public:
|
||||
// INTEGRATION: real null_sink() returns NullSink&, not shared_ptr, and
|
||||
// real counter() takes (name, int64, labels-string).
|
||||
explicit Appender(Store& store, ::molo::data::obs::MetricSink& metrics =
|
||||
::molo::data::obs::null_sink())
|
||||
: store_(store), metrics_(metrics) {}
|
||||
|
||||
struct Write {
|
||||
std::string actor;
|
||||
std::string action;
|
||||
std::string table;
|
||||
std::string pk;
|
||||
std::string op;
|
||||
std::string before_json;
|
||||
std::string after_json;
|
||||
std::string tenant;
|
||||
std::string request_id;
|
||||
std::string trace_id;
|
||||
};
|
||||
|
||||
// Appends one record, returns its seq. Throws on oversize record.
|
||||
std::uint64_t Append(Write w) {
|
||||
if (w.before_json.size() > kMaxBeforeBytes) w.before_json.resize(kMaxBeforeBytes);
|
||||
if (w.after_json.size() > kMaxAfterBytes) w.after_json.resize(kMaxAfterBytes);
|
||||
AuditRecord rec;
|
||||
rec.ts = Now();
|
||||
rec.tenant = std::move(w.tenant);
|
||||
rec.actor = std::move(w.actor);
|
||||
rec.action = std::move(w.action);
|
||||
rec.table = std::move(w.table);
|
||||
rec.pk = std::move(w.pk);
|
||||
rec.op = std::move(w.op);
|
||||
rec.before_json = std::move(w.before_json);
|
||||
rec.after_json = std::move(w.after_json);
|
||||
rec.request_id = std::move(w.request_id);
|
||||
rec.trace_id = std::move(w.trace_id);
|
||||
if (rec.Canonical().size() > kMaxRecordBytes)
|
||||
throw std::runtime_error("audit record exceeds kMaxRecordBytes");
|
||||
std::uint64_t seq = store_.Append(rec);
|
||||
appended_.fetch_add(1);
|
||||
metrics_.counter("audit_records_appended_total", 1, "table=" + rec.table);
|
||||
MaybeSeal(seq);
|
||||
return seq;
|
||||
}
|
||||
|
||||
// Process-wide writer installed once at deploy composition. Null until set.
|
||||
static void Install(Appender* a) { Global() = a; }
|
||||
static Appender* GlobalInstance() { return Global(); }
|
||||
std::uint64_t appended() const { return appended_.load(); }
|
||||
|
||||
private:
|
||||
static Appender*& Global() {
|
||||
static Appender* g = nullptr;
|
||||
return g;
|
||||
}
|
||||
|
||||
static std::int64_t Now() {
|
||||
return std::chrono::duration_cast<std::chrono::seconds>(
|
||||
std::chrono::system_clock::now().time_since_epoch())
|
||||
.count();
|
||||
}
|
||||
|
||||
void MaybeSeal(std::uint64_t seq) {
|
||||
if (seq % kSealEveryRecords != 0) return;
|
||||
Seal s = Chain::MakeSeal(seq / kSealEveryRecords, store_.Tip(), Now(), seal_key_);
|
||||
store_.SaveSeal(s);
|
||||
metrics_.counter("audit_seals_total", 1);
|
||||
}
|
||||
|
||||
Store& store_;
|
||||
::molo::data::obs::MetricSink& metrics_;
|
||||
std::string seal_key_; // empty until deploy composition provisions the key
|
||||
std::atomic<std::uint64_t> appended_{0};
|
||||
};
|
||||
|
||||
// ORM hook called from Model save()/remove() after bus->publish. No-op until
|
||||
// Appender::Install wires the store, so ORM boots without audit configured.
|
||||
inline void notify_write(const std::string& table, const std::string& pk, const std::string& op,
|
||||
const std::string& before_json, const std::string& after_json,
|
||||
const ::molo::data::Context& ctx, const std::string& actor = "") {
|
||||
Appender* a = Appender::GlobalInstance();
|
||||
if (!a) return;
|
||||
Appender::Write w;
|
||||
w.actor = actor;
|
||||
w.action = "orm." + op;
|
||||
w.table = table;
|
||||
w.pk = pk;
|
||||
w.op = op;
|
||||
w.before_json = before_json;
|
||||
w.after_json = after_json;
|
||||
w.tenant = ctx.tenant();
|
||||
w.request_id = ctx.correlation_id();
|
||||
a->Append(std::move(w));
|
||||
}
|
||||
|
||||
} // namespace molo::audit::log
|
||||
|
||||
// AUDIT(store, table, pk, op, before, after, ctx, actor): one-line ORM hook
|
||||
// filling tenant + request id from the data Context. Availability of the
|
||||
// global Appender decides whether this writes or no-ops.
|
||||
#define AUDIT(store, table, pk, op, before, after, ctx, actor) \
|
||||
::molo::audit::log::notify_write((table), (pk), (op), (before), (after), (ctx), (actor))
|
||||
@@ -0,0 +1,112 @@
|
||||
// Hash chain over audit records: hash = SHA256hex(prev_hash || canonical).
|
||||
// Genesis prev_hash is 64 zeros. Seal checkpoints the tip with an HMAC under
|
||||
// the audit seal key; Verify replays the chain and names the exact sequence
|
||||
// number of the first gap or tampered record.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include "../adapt.h"
|
||||
#include "../caps.h"
|
||||
#include "record.h"
|
||||
|
||||
namespace molo::audit::log {
|
||||
|
||||
inline const std::string& GenesisHash() {
|
||||
static const std::string k(64, '0');
|
||||
return k;
|
||||
}
|
||||
|
||||
struct Seal {
|
||||
std::uint64_t seal_seq = 0;
|
||||
std::string root; // tip hash at seal time
|
||||
std::int64_t ts = 0;
|
||||
std::string mac; // HMAC-SHA256 hex over "seal_seq:root:ts" under seal key
|
||||
};
|
||||
|
||||
struct VerifyResult {
|
||||
bool ok = true;
|
||||
std::uint64_t bad_seq = 0; // first gap or tamper, valid when !ok
|
||||
std::string reason; // "gap", "prev", "hash", "seal"
|
||||
};
|
||||
|
||||
class Chain {
|
||||
public:
|
||||
static std::string HashEntry(std::string_view prev_hash, std::string_view canonical) {
|
||||
std::string in(prev_hash);
|
||||
in += canonical;
|
||||
return ::molo::security::crypto::Sha256::hex(
|
||||
::molo::security::crypto::Sha256::hash(in));
|
||||
}
|
||||
|
||||
// Stamps prev/hash onto rec given the previous tip (genesis for seq 1).
|
||||
static void Link(AuditRecord& rec, std::string_view prev_tip) {
|
||||
rec.prev_hash = std::string(prev_tip);
|
||||
rec.hash = HashEntry(rec.prev_hash, rec.Canonical());
|
||||
}
|
||||
|
||||
static Seal MakeSeal(std::uint64_t seal_seq, std::string_view root, std::int64_t ts,
|
||||
std::string_view seal_key) {
|
||||
Seal s;
|
||||
s.seal_seq = seal_seq;
|
||||
s.root = std::string(root);
|
||||
s.ts = ts;
|
||||
std::string body =
|
||||
std::to_string(seal_seq) + ":" + s.root + ":" + std::to_string(ts);
|
||||
// INTEGRATION: real Hmac takes (const string&, const string&) and hex
|
||||
// takes a non-const ref, so materialize both arguments.
|
||||
auto mac = ::molo::security::crypto::Hmac::sha256(std::string(seal_key), body);
|
||||
s.mac = ::molo::security::crypto::Hmac::hex(mac);
|
||||
return s;
|
||||
}
|
||||
|
||||
static bool CheckSeal(const Seal& s, std::string_view seal_key) {
|
||||
return MakeSeal(s.seal_seq, s.root, s.ts, seal_key).mac == s.mac;
|
||||
}
|
||||
|
||||
// Replays ordered records: seq continuity, prev linkage, entry hashes.
|
||||
// When seals is non-empty the covered tips must match a valid seal root.
|
||||
static VerifyResult Verify(const std::vector<AuditRecord>& recs,
|
||||
const std::vector<Seal>& seals = {},
|
||||
std::string_view seal_key = "") {
|
||||
VerifyResult r;
|
||||
std::string tip = GenesisHash();
|
||||
std::uint64_t expect = recs.empty() ? 0 : recs.front().seq;
|
||||
for (const auto& rec : recs) {
|
||||
if (rec.seq != expect) {
|
||||
r.ok = false;
|
||||
r.bad_seq = expect;
|
||||
r.reason = "gap";
|
||||
return r;
|
||||
}
|
||||
if (rec.prev_hash != tip) {
|
||||
r.ok = false;
|
||||
r.bad_seq = rec.seq;
|
||||
r.reason = "prev";
|
||||
return r;
|
||||
}
|
||||
if (rec.hash != HashEntry(rec.prev_hash, rec.Canonical())) {
|
||||
r.ok = false;
|
||||
r.bad_seq = rec.seq;
|
||||
r.reason = "hash";
|
||||
return r;
|
||||
}
|
||||
tip = rec.hash;
|
||||
++expect;
|
||||
}
|
||||
for (const auto& s : seals) {
|
||||
if (!seal_key.empty() && !CheckSeal(s, seal_key)) {
|
||||
r.ok = false;
|
||||
r.reason = "seal";
|
||||
return r;
|
||||
}
|
||||
}
|
||||
(void)kSealEveryRecords;
|
||||
return r;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace molo::audit::log
|
||||
@@ -0,0 +1,107 @@
|
||||
// Query: filtered, paged reads over the audit trail. Every query requires
|
||||
// the audit.read grant (Guard verdict 403 otherwise) and every query, allowed
|
||||
// or denied, appends its own self-audit record so trail reads stay visible.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "../adapt.h"
|
||||
#include "../caps.h"
|
||||
#include "appender.h"
|
||||
#include "record.h"
|
||||
#include "store.h"
|
||||
|
||||
namespace molo::audit::log {
|
||||
|
||||
struct Filter {
|
||||
std::string tenant;
|
||||
std::string table;
|
||||
std::string actor;
|
||||
std::string op;
|
||||
std::uint64_t from_seq = 0;
|
||||
std::uint64_t to_seq = 0; // 0 means no upper bound
|
||||
std::size_t limit = 100;
|
||||
std::string cursor; // seq to resume after, empty starts at from_seq
|
||||
};
|
||||
|
||||
struct QueryPage {
|
||||
int http_status = 200;
|
||||
std::string code = "ok";
|
||||
std::vector<AuditRecord> rows;
|
||||
std::string next_cursor;
|
||||
};
|
||||
|
||||
class Query {
|
||||
public:
|
||||
Query(Store& store, Appender* audit) : store_(store), audit_(audit) {}
|
||||
|
||||
// INTEGRATION: real Guard::check takes (Session, roles, required) with
|
||||
// Session from security/session; any denial maps to 403 per this API.
|
||||
QueryPage Run(const Filter& f, const ::molo::security::session::Session& session,
|
||||
const std::vector<::molo::security::auth::Role>& roles = {}) {
|
||||
using ::molo::security::auth::Guard;
|
||||
using ::molo::security::auth::Permission;
|
||||
VerdictOf(Guard::check(session, roles, {Permission{"audit.read"}}));
|
||||
if (last_status_ == 403) {
|
||||
SelfAudit(session, f, "query.denied");
|
||||
QueryPage p;
|
||||
p.http_status = 403;
|
||||
p.code = "forbidden";
|
||||
return p;
|
||||
}
|
||||
std::size_t limit = f.limit == 0 ? 100 : f.limit;
|
||||
if (limit > kQueryPageMax) limit = kQueryPageMax;
|
||||
std::uint64_t start = f.from_seq == 0 ? 1 : f.from_seq;
|
||||
if (!f.cursor.empty()) start = std::stoull(f.cursor) + 1;
|
||||
QueryPage page;
|
||||
std::uint64_t probe = start;
|
||||
while (page.rows.size() < limit) {
|
||||
auto batch = store_.ReadRange(probe, limit - page.rows.size());
|
||||
if (batch.empty()) break;
|
||||
for (auto& r : batch) {
|
||||
probe = r.seq + 1;
|
||||
if (f.to_seq != 0 && r.seq > f.to_seq) {
|
||||
probe = 0;
|
||||
break;
|
||||
}
|
||||
if (!f.tenant.empty() && r.tenant != f.tenant) continue;
|
||||
if (!f.table.empty() && r.table != f.table) continue;
|
||||
if (!f.actor.empty() && r.actor != f.actor) continue;
|
||||
if (!f.op.empty() && r.op != f.op) continue;
|
||||
page.rows.push_back(r);
|
||||
if (page.rows.size() >= limit) break;
|
||||
}
|
||||
if (probe == 0) break;
|
||||
}
|
||||
if (!page.rows.empty()) page.next_cursor = std::to_string(page.rows.back().seq);
|
||||
SelfAudit(session, f, "query.allowed");
|
||||
return page;
|
||||
}
|
||||
|
||||
private:
|
||||
void VerdictOf(const ::molo::security::auth::Guard::Verdict& v) {
|
||||
last_status_ = v.allowed ? 200 : 403;
|
||||
}
|
||||
|
||||
void SelfAudit(const ::molo::security::session::Session& s, const Filter& f,
|
||||
const std::string& action) {
|
||||
if (!audit_) return;
|
||||
Appender::Write w;
|
||||
w.actor = s.user_id;
|
||||
w.action = action;
|
||||
w.table = "audit_records";
|
||||
w.op = "read";
|
||||
w.tenant = f.tenant;
|
||||
w.after_json = "{\"table\":\"" + AuditRecord::Escape(f.table) + "\"}";
|
||||
audit_->Append(std::move(w));
|
||||
}
|
||||
|
||||
Store& store_;
|
||||
Appender* audit_;
|
||||
int last_status_ = 200;
|
||||
};
|
||||
|
||||
} // namespace molo::audit::log
|
||||
@@ -0,0 +1,81 @@
|
||||
// AuditRecord: one immutable audit trail entry with fixed JSON keys.
|
||||
// Key order is frozen (spec 6.1): v,seq,ts,tenant,actor,action,table,pk,op,
|
||||
// before,after,req,trace,prev,hash. Canonical() renders exactly those keys
|
||||
// so the hash chain is stable across backends and releases.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
namespace molo::audit::log {
|
||||
|
||||
struct AuditRecord {
|
||||
std::uint64_t seq = 0;
|
||||
std::int64_t ts = 0; // unix seconds
|
||||
std::string tenant;
|
||||
std::string actor;
|
||||
std::string action;
|
||||
std::string table;
|
||||
std::string pk;
|
||||
std::string op; // insert, update, delete, read, export, erase, seal
|
||||
std::string before_json;
|
||||
std::string after_json;
|
||||
std::string request_id;
|
||||
std::string trace_id;
|
||||
std::string prev_hash; // 64 hex chars, genesis is 64 zeros
|
||||
std::string hash; // 64 hex chars over prev_hash + canonical body
|
||||
|
||||
static std::string Escape(std::string_view s) {
|
||||
std::string o;
|
||||
o.reserve(s.size() + 2);
|
||||
for (char c : s) {
|
||||
switch (c) {
|
||||
case '"':
|
||||
o += "\\\"";
|
||||
break;
|
||||
case '\\':
|
||||
o += "\\\\";
|
||||
break;
|
||||
case '\n':
|
||||
o += "\\n";
|
||||
break;
|
||||
case '\r':
|
||||
o += "\\r";
|
||||
break;
|
||||
case '\t':
|
||||
o += "\\t";
|
||||
break;
|
||||
default:
|
||||
o += c;
|
||||
}
|
||||
}
|
||||
return o;
|
||||
}
|
||||
|
||||
// Canonical body: every key except hash, fixed order, no whitespace.
|
||||
std::string Canonical() const {
|
||||
std::string o = "{\"v\":1";
|
||||
o += ",\"seq\":" + std::to_string(seq);
|
||||
o += ",\"ts\":" + std::to_string(ts);
|
||||
o += ",\"tenant\":\"" + Escape(tenant) + "\"";
|
||||
o += ",\"actor\":\"" + Escape(actor) + "\"";
|
||||
o += ",\"action\":\"" + Escape(action) + "\"";
|
||||
o += ",\"table\":\"" + Escape(table) + "\"";
|
||||
o += ",\"pk\":\"" + Escape(pk) + "\"";
|
||||
o += ",\"op\":\"" + Escape(op) + "\"";
|
||||
o += ",\"before\":" + (before_json.empty() ? "null" : before_json);
|
||||
o += ",\"after\":" + (after_json.empty() ? "null" : after_json);
|
||||
o += ",\"req\":\"" + Escape(request_id) + "\"";
|
||||
o += ",\"trace\":\"" + Escape(trace_id) + "\"";
|
||||
o += ",\"prev\":\"" + Escape(prev_hash) + "\"}";
|
||||
return o;
|
||||
}
|
||||
|
||||
std::string ToJson() const {
|
||||
std::string c = Canonical();
|
||||
c.pop_back(); // strip closing brace
|
||||
return c + ",\"hash\":\"" + Escape(hash) + "\"}";
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace molo::audit::log
|
||||
@@ -0,0 +1,216 @@
|
||||
// SqlStore: production Store over both SQL backends (Postgres, SQLite).
|
||||
// Tables audit_records and audit_seals use one portable DDL; only the
|
||||
// parameter placeholder style differs per backend. Binding goes through the
|
||||
// tiny SqlExec seam so this header compiles standalone; ConnectionExec binds
|
||||
// it to a real molo::data::Connection, tests bind a FakeExec.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "../adapt.h"
|
||||
#include "../caps.h"
|
||||
#include "chain.h"
|
||||
#include "record.h"
|
||||
#include "store.h"
|
||||
|
||||
namespace molo::audit::log {
|
||||
|
||||
// Minimal SQL surface the audit store needs. Params are strings; the
|
||||
// Connection binding promotes integer-looking params to make_int.
|
||||
class SqlExec {
|
||||
public:
|
||||
virtual ~SqlExec() = default;
|
||||
virtual void Exec(const std::string& sql, const std::vector<std::string>& params) = 0;
|
||||
virtual std::vector<std::vector<std::string>> Query(const std::string& sql,
|
||||
const std::vector<std::string>& params,
|
||||
std::size_t max_rows) = 0;
|
||||
virtual bool IsPostgres() const = 0;
|
||||
};
|
||||
|
||||
// Binds SqlExec to a real framework Connection. Both backends supported.
|
||||
class ConnectionExec : public SqlExec {
|
||||
public:
|
||||
ConnectionExec(::molo::data::Connection& c, ::molo::data::Context ctx)
|
||||
: conn_(c), ctx_(std::move(ctx)) {}
|
||||
|
||||
void Exec(const std::string& sql, const std::vector<std::string>& params) override {
|
||||
conn_.exec(ctx_, ::molo::data::BoundStatement(sql, ToValues(params)));
|
||||
}
|
||||
|
||||
std::vector<std::vector<std::string>> Query(const std::string& sql,
|
||||
const std::vector<std::string>& params,
|
||||
std::size_t max_rows) override {
|
||||
auto res = conn_.exec(ctx_, ::molo::data::BoundStatement(sql, ToValues(params)));
|
||||
std::vector<std::vector<std::string>> out;
|
||||
// INTEGRATION: real Result exposes size()/raw(i), not a rows member.
|
||||
for (std::size_t i = 0; i < res.size(); ++i) {
|
||||
if (out.size() >= max_rows) break;
|
||||
std::vector<std::string> r;
|
||||
for (auto& v : res.raw(i)) r.push_back(FromValue(v));
|
||||
out.push_back(std::move(r));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
bool IsPostgres() const override { return conn_.is_postgres(); }
|
||||
|
||||
private:
|
||||
static std::vector<::molo::data::Value> ToValues(const std::vector<std::string>& p) {
|
||||
std::vector<::molo::data::Value> v;
|
||||
for (auto& s : p) {
|
||||
char* end = nullptr;
|
||||
long long n = std::strtoll(s.c_str(), &end, 10);
|
||||
if (end && *end == '\0' && end != s.c_str())
|
||||
v.push_back(::molo::data::make_int(n));
|
||||
else
|
||||
v.push_back(::molo::data::make_text(s));
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
static std::string FromValue(const ::molo::data::Value& v) {
|
||||
if (auto* i = std::get_if<std::int64_t>(&v)) return std::to_string(*i);
|
||||
if (auto* s = std::get_if<std::string>(&v)) return *s;
|
||||
if (auto* d = std::get_if<double>(&v)) return std::to_string(*d);
|
||||
if (auto* b = std::get_if<bool>(&v)) return *b ? "1" : "0";
|
||||
return "";
|
||||
}
|
||||
|
||||
::molo::data::Connection& conn_;
|
||||
::molo::data::Context ctx_;
|
||||
};
|
||||
|
||||
class SqlStore : public Store {
|
||||
public:
|
||||
explicit SqlStore(SqlExec& db) : db_(db) { EnsureSchema(); }
|
||||
|
||||
static std::string DdlRecords() {
|
||||
return "CREATE TABLE IF NOT EXISTS audit_records (seq INTEGER PRIMARY KEY,"
|
||||
"ts BIGINT NOT NULL,tenant TEXT NOT NULL,actor TEXT NOT NULL,"
|
||||
"action TEXT NOT NULL,tbl TEXT NOT NULL,pk TEXT NOT NULL,"
|
||||
"op TEXT NOT NULL,before_json TEXT NOT NULL,after_json TEXT NOT NULL,"
|
||||
"req TEXT NOT NULL,trace TEXT NOT NULL,prev_hash TEXT NOT NULL,"
|
||||
"hash TEXT NOT NULL)";
|
||||
}
|
||||
|
||||
static std::string DdlSeals() {
|
||||
return "CREATE TABLE IF NOT EXISTS audit_seals (seal_seq INTEGER PRIMARY KEY,"
|
||||
"root TEXT NOT NULL,ts BIGINT NOT NULL,mac TEXT NOT NULL)";
|
||||
}
|
||||
|
||||
std::uint64_t Append(AuditRecord rec) override {
|
||||
std::lock_guard<std::mutex> g(mu_);
|
||||
auto tip_rows =
|
||||
db_.Query("SELECT seq,hash FROM audit_records ORDER BY seq DESC LIMIT 1", {}, 1);
|
||||
std::uint64_t seq = 1;
|
||||
std::string tip = GenesisHash();
|
||||
if (!tip_rows.empty() && tip_rows[0].size() == 2) {
|
||||
seq = std::stoull(tip_rows[0][0]) + 1;
|
||||
tip = tip_rows[0][1];
|
||||
}
|
||||
rec.seq = seq;
|
||||
Chain::Link(rec, tip);
|
||||
const bool pg = db_.IsPostgres();
|
||||
std::string sql =
|
||||
"INSERT INTO audit_records (seq,ts,tenant,actor,action,tbl,pk,op,before_json,"
|
||||
"after_json,req,trace,prev_hash,hash) VALUES (" +
|
||||
Placeholders(14, pg) + ")";
|
||||
db_.Exec(sql, {std::to_string(rec.seq), std::to_string(rec.ts), rec.tenant, rec.actor,
|
||||
rec.action, rec.table, rec.pk, rec.op, rec.before_json, rec.after_json,
|
||||
rec.request_id, rec.trace_id, rec.prev_hash, rec.hash});
|
||||
return seq;
|
||||
}
|
||||
|
||||
std::vector<AuditRecord> ReadRange(std::uint64_t from_seq, std::size_t limit) override {
|
||||
std::lock_guard<std::mutex> g(mu_);
|
||||
if (limit > kQueryPageMax) limit = kQueryPageMax;
|
||||
const bool pg = db_.IsPostgres();
|
||||
std::string sql =
|
||||
"SELECT seq,ts,tenant,actor,action,tbl,pk,op,before_json,after_json,req,trace,"
|
||||
"prev_hash,hash FROM audit_records WHERE seq >= " +
|
||||
Ph(1, pg) + " ORDER BY seq ASC LIMIT " + std::to_string(limit);
|
||||
std::vector<AuditRecord> out;
|
||||
for (auto& r : db_.Query(sql, {std::to_string(from_seq)}, limit)) {
|
||||
if (r.size() != 14) continue;
|
||||
AuditRecord rec;
|
||||
rec.seq = std::stoull(r[0]);
|
||||
rec.ts = std::stoll(r[1]);
|
||||
rec.tenant = r[2];
|
||||
rec.actor = r[3];
|
||||
rec.action = r[4];
|
||||
rec.table = r[5];
|
||||
rec.pk = r[6];
|
||||
rec.op = r[7];
|
||||
rec.before_json = r[8];
|
||||
rec.after_json = r[9];
|
||||
rec.request_id = r[10];
|
||||
rec.trace_id = r[11];
|
||||
rec.prev_hash = r[12];
|
||||
rec.hash = r[13];
|
||||
out.push_back(std::move(rec));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
void SaveSeal(const Seal& s) override {
|
||||
std::lock_guard<std::mutex> g(mu_);
|
||||
const bool pg = db_.IsPostgres();
|
||||
db_.Exec("INSERT INTO audit_seals (seal_seq,root,ts,mac) VALUES (" + Placeholders(4, pg) +
|
||||
")",
|
||||
{std::to_string(s.seal_seq), s.root, std::to_string(s.ts), s.mac});
|
||||
}
|
||||
|
||||
std::vector<Seal> Seals() override {
|
||||
std::lock_guard<std::mutex> g(mu_);
|
||||
std::vector<Seal> out;
|
||||
for (auto& r : db_.Query("SELECT seal_seq,root,ts,mac FROM audit_seals ORDER BY seal_seq ASC",
|
||||
{}, 100000)) {
|
||||
if (r.size() != 4) continue;
|
||||
Seal s;
|
||||
s.seal_seq = std::stoull(r[0]);
|
||||
s.root = r[1];
|
||||
s.ts = std::stoll(r[2]);
|
||||
s.mac = r[3];
|
||||
out.push_back(std::move(s));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::uint64_t LatestSeq() override {
|
||||
std::lock_guard<std::mutex> g(mu_);
|
||||
auto rows = db_.Query("SELECT seq FROM audit_records ORDER BY seq DESC LIMIT 1", {}, 1);
|
||||
return rows.empty() ? 0 : std::stoull(rows[0][0]);
|
||||
}
|
||||
|
||||
std::string Tip() override {
|
||||
std::lock_guard<std::mutex> g(mu_);
|
||||
auto rows = db_.Query("SELECT hash FROM audit_records ORDER BY seq DESC LIMIT 1", {}, 1);
|
||||
return rows.empty() ? GenesisHash() : rows[0][0];
|
||||
}
|
||||
|
||||
private:
|
||||
void EnsureSchema() {
|
||||
db_.Exec(DdlRecords(), {});
|
||||
db_.Exec(DdlSeals(), {});
|
||||
}
|
||||
|
||||
static std::string Ph(int n, bool pg) { return pg ? "$" + std::to_string(n) : "?"; }
|
||||
|
||||
static std::string Placeholders(int n, bool pg) {
|
||||
std::string o;
|
||||
for (int i = 1; i <= n; ++i) {
|
||||
if (i > 1) o += ",";
|
||||
o += Ph(i, pg);
|
||||
}
|
||||
return o;
|
||||
}
|
||||
|
||||
SqlExec& db_;
|
||||
std::mutex mu_;
|
||||
};
|
||||
|
||||
} // namespace molo::audit::log
|
||||
@@ -0,0 +1,80 @@
|
||||
// Store: the only persistence seam for the audit trail. Three operations:
|
||||
// append, read_range, seal. MemoryStore is the standalone/test backend;
|
||||
// SqlStore (sql_store.h) is the production backend over both SQL drivers.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "../caps.h"
|
||||
#include "chain.h"
|
||||
#include "record.h"
|
||||
|
||||
namespace molo::audit::log {
|
||||
|
||||
class Store {
|
||||
public:
|
||||
virtual ~Store() = default;
|
||||
// Appends rec, assigns rec.seq, links the chain, returns the seq.
|
||||
virtual std::uint64_t Append(AuditRecord rec) = 0;
|
||||
// Reads at most limit records from from_seq (inclusive), ascending.
|
||||
virtual std::vector<AuditRecord> ReadRange(std::uint64_t from_seq, std::size_t limit) = 0;
|
||||
// Persists a chain seal checkpoint.
|
||||
virtual void SaveSeal(const Seal& s) = 0;
|
||||
virtual std::vector<Seal> Seals() = 0;
|
||||
virtual std::uint64_t LatestSeq() = 0;
|
||||
// Current chain tip, genesis when empty.
|
||||
virtual std::string Tip() = 0;
|
||||
};
|
||||
|
||||
class MemoryStore : public Store {
|
||||
public:
|
||||
std::uint64_t Append(AuditRecord rec) override {
|
||||
std::lock_guard<std::mutex> g(mu_);
|
||||
rec.seq = recs_.size() + 1;
|
||||
Chain::Link(rec, tip_);
|
||||
tip_ = rec.hash;
|
||||
recs_.push_back(std::move(rec));
|
||||
return recs_.size();
|
||||
}
|
||||
|
||||
std::vector<AuditRecord> ReadRange(std::uint64_t from_seq, std::size_t limit) override {
|
||||
std::lock_guard<std::mutex> g(mu_);
|
||||
if (limit > kQueryPageMax) limit = kQueryPageMax;
|
||||
std::vector<AuditRecord> out;
|
||||
if (from_seq == 0 || from_seq > recs_.size()) return out;
|
||||
for (std::size_t i = from_seq - 1; i < recs_.size() && out.size() < limit; ++i)
|
||||
out.push_back(recs_[i]);
|
||||
return out;
|
||||
}
|
||||
|
||||
void SaveSeal(const Seal& s) override {
|
||||
std::lock_guard<std::mutex> g(mu_);
|
||||
seals_.push_back(s);
|
||||
}
|
||||
|
||||
std::vector<Seal> Seals() override {
|
||||
std::lock_guard<std::mutex> g(mu_);
|
||||
return seals_;
|
||||
}
|
||||
|
||||
std::uint64_t LatestSeq() override {
|
||||
std::lock_guard<std::mutex> g(mu_);
|
||||
return recs_.size();
|
||||
}
|
||||
|
||||
std::string Tip() override {
|
||||
std::lock_guard<std::mutex> g(mu_);
|
||||
return tip_;
|
||||
}
|
||||
|
||||
private:
|
||||
std::mutex mu_;
|
||||
std::vector<AuditRecord> recs_;
|
||||
std::vector<Seal> seals_;
|
||||
std::string tip_ = GenesisHash();
|
||||
};
|
||||
|
||||
} // namespace molo::audit::log
|
||||
@@ -0,0 +1,84 @@
|
||||
// Fixture scrubber: streams fixture blobs through in 1MB chunks, masking
|
||||
// every PII hit with the Masker, then rescans the output and reports any
|
||||
// residual raw values. Chunks overlap by 1024 bytes (twice the max match
|
||||
// length) so matches split across a boundary are still caught whole.
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include "../caps.h"
|
||||
#include "../redact/pattern.h"
|
||||
#include "masker.h"
|
||||
|
||||
namespace molo::audit::mask {
|
||||
|
||||
struct ScrubStats {
|
||||
std::uint64_t chunks = 0;
|
||||
std::uint64_t masked = 0;
|
||||
std::uint64_t residual = 0; // raw hits surviving the rescan, must be 0
|
||||
};
|
||||
|
||||
class Fixture {
|
||||
public:
|
||||
static constexpr std::size_t kOverlap = 1024;
|
||||
|
||||
using EmitFn = std::function<void(std::string_view)>;
|
||||
|
||||
// Scrubs blob in kFixtureChunkBytes pieces, emitting masked output.
|
||||
static ScrubStats Scrub(std::string_view blob, const Masker& masker, EmitFn emit) {
|
||||
ScrubStats st;
|
||||
std::size_t pos = 0;
|
||||
std::string carry;
|
||||
while (pos < blob.size()) {
|
||||
std::size_t n = kFixtureChunkBytes;
|
||||
if (n > blob.size() - pos) n = blob.size() - pos;
|
||||
std::string window = carry + std::string(blob.substr(pos, n));
|
||||
pos += n;
|
||||
++st.chunks;
|
||||
std::string masked = MaskWindow(window, masker, st);
|
||||
if (pos < blob.size() && masked.size() > kOverlap) {
|
||||
// Hold the overlap back so boundary matches rescan in full context.
|
||||
std::string hold = masked.substr(masked.size() - kOverlap);
|
||||
emit(std::string_view(masked.data(), masked.size() - kOverlap));
|
||||
carry = hold;
|
||||
} else {
|
||||
emit(masked);
|
||||
carry.clear();
|
||||
}
|
||||
}
|
||||
return st;
|
||||
}
|
||||
|
||||
// Counts raw PII pattern hits left in scrubbed output. Zero means clean.
|
||||
// Masked tokens ("masked:<class>:<hex>") never match a scanner shape, so
|
||||
// every hit counted here is a genuine residual.
|
||||
static std::uint64_t Rescan(std::string_view scrubbed) {
|
||||
return redact::FindHits(scrubbed).size();
|
||||
}
|
||||
|
||||
private:
|
||||
static std::string MaskWindow(const std::string& window, const Masker& masker,
|
||||
ScrubStats& st) {
|
||||
std::vector<redact::Hit> hits = redact::FindHits(window);
|
||||
std::sort(hits.begin(), hits.end(),
|
||||
[](const redact::Hit& a, const redact::Hit& b) { return a.pos < b.pos; });
|
||||
std::string out;
|
||||
out.reserve(window.size() + hits.size() * 8);
|
||||
std::size_t last = 0;
|
||||
for (auto& h : hits) {
|
||||
out.append(window, last, h.pos - last);
|
||||
out += masker.Mask(window.substr(h.pos, h.len), h.cls);
|
||||
last = h.pos + h.len;
|
||||
++st.masked;
|
||||
}
|
||||
out.append(window, last, std::string::npos);
|
||||
return out;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace molo::audit::mask
|
||||
@@ -0,0 +1,53 @@
|
||||
// Masker: seeded deterministic masking. Same (seed, value, class) always
|
||||
// yields the same mask, so masked fixtures keep working foreign-key joins
|
||||
// across tables and runs. Masks carry no recoverable prefix of the input.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
#include "../adapt.h"
|
||||
#include "../pii/mark.h"
|
||||
|
||||
namespace molo::audit::mask {
|
||||
|
||||
class Masker {
|
||||
public:
|
||||
explicit Masker(std::string seed) : seed_(std::move(seed)) {}
|
||||
|
||||
// Deterministic mask for one value. Format keeps a class tag plus a
|
||||
// 16-char body derived from seed + class + value, e.g. "masked:email:..".
|
||||
// Every 4th body char is forced to a letter, so no mask ever contains a
|
||||
// digit run the phone/card rescanners could flag: masks are rescan-proof
|
||||
// by construction, for every seed.
|
||||
std::string Mask(std::string_view value, pii::PiiClass cls) const {
|
||||
std::string in = seed_ + ":" + pii::ToString(cls) + ":";
|
||||
in += value;
|
||||
std::string hex = ::molo::security::crypto::Sha256::hex(
|
||||
::molo::security::crypto::Sha256::hash(in));
|
||||
return "masked:" + pii::ToString(cls) + ":" + Body(hex);
|
||||
}
|
||||
|
||||
// Deterministic join key: stable across tables for the same raw id.
|
||||
std::string JoinKey(std::string_view raw_id) const {
|
||||
std::string in = seed_ + ":join:";
|
||||
in += raw_id;
|
||||
std::string hex = ::molo::security::crypto::Sha256::hex(
|
||||
::molo::security::crypto::Sha256::hash(in));
|
||||
return "join:" + Body(hex);
|
||||
}
|
||||
|
||||
private:
|
||||
static std::string Body(const std::string& hex) {
|
||||
static constexpr char kForce[] = {'g', 'h', 'j', 'k'};
|
||||
std::string b = hex.substr(0, 16);
|
||||
for (std::size_t i = 3; i < b.size(); i += 4)
|
||||
if (b[i] >= '0' && b[i] <= '9') b[i] = kForce[(b[i] - '0') % 4];
|
||||
return b;
|
||||
}
|
||||
|
||||
std::string seed_;
|
||||
};
|
||||
|
||||
} // namespace molo::audit::mask
|
||||
@@ -0,0 +1,68 @@
|
||||
// Field-level PII classification mark. Attached to ORM FieldDef by the base
|
||||
// patch (see PATCHES.md entry 1); consumed by Registry, Redactor, Exporter,
|
||||
// Eraser and Masker so every component agrees on what a field holds.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
namespace molo::audit::pii {
|
||||
|
||||
// Ten stable classes. Numeric values are persisted in marks and fixtures,
|
||||
// never renumber, only append.
|
||||
enum class PiiClass : std::uint8_t {
|
||||
kNone = 0,
|
||||
kName = 1,
|
||||
kEmail = 2,
|
||||
kPhone = 3,
|
||||
kAddress = 4,
|
||||
kNationalId = 5,
|
||||
kFinancial = 6,
|
||||
kHealth = 7,
|
||||
kCredential = 8,
|
||||
kLocation = 9,
|
||||
};
|
||||
|
||||
inline std::string ToString(PiiClass c) {
|
||||
switch (c) {
|
||||
case PiiClass::kNone:
|
||||
return "none";
|
||||
case PiiClass::kName:
|
||||
return "name";
|
||||
case PiiClass::kEmail:
|
||||
return "email";
|
||||
case PiiClass::kPhone:
|
||||
return "phone";
|
||||
case PiiClass::kAddress:
|
||||
return "address";
|
||||
case PiiClass::kNationalId:
|
||||
return "national_id";
|
||||
case PiiClass::kFinancial:
|
||||
return "financial";
|
||||
case PiiClass::kHealth:
|
||||
return "health";
|
||||
case PiiClass::kCredential:
|
||||
return "credential";
|
||||
case PiiClass::kLocation:
|
||||
return "location";
|
||||
}
|
||||
return "none";
|
||||
}
|
||||
|
||||
// What erasure must do with a marked field.
|
||||
enum class ErasureRule : std::uint8_t {
|
||||
kKeep = 0, // Legal hold or statutory retention: never erase.
|
||||
kMask = 1, // Replace with deterministic mask, keep row shape.
|
||||
kDelete = 2, // Null the column or drop the row.
|
||||
};
|
||||
|
||||
// One field's compliance mark: its PII class, whether subject export may
|
||||
// include it, how erasure treats it, and the legal basis string.
|
||||
struct FieldMark {
|
||||
PiiClass cls = PiiClass::kNone;
|
||||
bool exportable = true;
|
||||
ErasureRule erasure = ErasureRule::kDelete;
|
||||
std::string basis;
|
||||
};
|
||||
|
||||
} // namespace molo::audit::pii
|
||||
@@ -0,0 +1,55 @@
|
||||
// Boot-time registry of every ORM field's PII mark. Build fails loudly when
|
||||
// any schema field lacks a mark, so unclassified columns can never reach
|
||||
// production silently.
|
||||
#pragma once
|
||||
|
||||
#include <map>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "mark.h"
|
||||
|
||||
namespace molo::audit::pii {
|
||||
|
||||
class Registry {
|
||||
public:
|
||||
// schema maps table -> its column names; marks maps "table.field" -> mark.
|
||||
// Throws std::runtime_error listing every unmarked field.
|
||||
static Registry Build(const std::map<std::string, std::vector<std::string>>& schema,
|
||||
const std::map<std::string, FieldMark>& marks) {
|
||||
Registry r;
|
||||
std::string missing;
|
||||
for (const auto& [table, cols] : schema) {
|
||||
for (const auto& col : cols) {
|
||||
std::string key = table + "." + col;
|
||||
auto it = marks.find(key);
|
||||
if (it == marks.end()) {
|
||||
if (!missing.empty()) missing += ",";
|
||||
missing += key;
|
||||
} else {
|
||||
r.marks_[key] = it->second;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!missing.empty()) throw std::runtime_error("unmarked pii fields: " + missing);
|
||||
return r;
|
||||
}
|
||||
|
||||
const FieldMark& Lookup(const std::string& table, const std::string& field) const {
|
||||
static const FieldMark kDefault;
|
||||
auto it = marks_.find(table + "." + field);
|
||||
return it == marks_.end() ? kDefault : it->second;
|
||||
}
|
||||
|
||||
bool Marked(const std::string& table, const std::string& field) const {
|
||||
return marks_.count(table + "." + field) > 0;
|
||||
}
|
||||
|
||||
std::size_t size() const { return marks_.size(); }
|
||||
|
||||
private:
|
||||
std::map<std::string, FieldMark> marks_;
|
||||
};
|
||||
|
||||
} // namespace molo::audit::pii
|
||||
@@ -0,0 +1,209 @@
|
||||
// Value-shape scanners that spot PII in free text (log messages, error
|
||||
// detail, fixture blobs). Hand-written matchers, no std::regex: libstdc++
|
||||
// regex recurses per input character and overflows the stack on large
|
||||
// windows. Each pattern names the PiiClass it detects so callers can emit
|
||||
// [redacted:<class>] or a deterministic mask. Matches are length-capped and
|
||||
// hits per call are capped, so hostile input cannot blow up time or memory.
|
||||
#pragma once
|
||||
|
||||
#include <cctype>
|
||||
#include <functional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include "../caps.h"
|
||||
#include "../pii/mark.h"
|
||||
|
||||
namespace molo::audit::redact {
|
||||
|
||||
inline constexpr std::size_t kMaxMatchLen = 512;
|
||||
|
||||
struct Hit {
|
||||
std::size_t pos = 0;
|
||||
std::size_t len = 0;
|
||||
pii::PiiClass cls = pii::PiiClass::kNone;
|
||||
};
|
||||
|
||||
namespace detail {
|
||||
inline bool IsEmailLocal(char c) {
|
||||
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') ||
|
||||
c == '.' || c == '_' || c == '%' || c == '+' || c == '-';
|
||||
}
|
||||
inline bool IsEmailDomain(char c) {
|
||||
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') ||
|
||||
c == '.' || c == '-';
|
||||
}
|
||||
inline bool IsAlpha(char c) {
|
||||
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
|
||||
}
|
||||
inline bool IsDigit(char c) { return c >= '0' && c <= '9'; }
|
||||
inline bool IsPhoneMid(char c) {
|
||||
return IsDigit(c) || c == ' ' || c == '.' || c == '-' || c == '(' || c == ')';
|
||||
}
|
||||
inline char Lower(char c) { return (c >= 'A' && c <= 'Z') ? c - 'A' + 'a' : c; }
|
||||
|
||||
inline void FindEmail(std::string_view t, std::vector<Hit>& out) {
|
||||
for (std::size_t at = t.find('@'); at != std::string_view::npos; at = t.find('@', at + 1)) {
|
||||
std::size_t l = at;
|
||||
while (l > 0 && IsEmailLocal(t[l - 1]) && at - l < 64) --l;
|
||||
if (l == at || (at > 0 && IsEmailLocal(t[l - 1]) && at - l == 64)) continue;
|
||||
std::size_t r = at + 1;
|
||||
while (r < t.size() && IsEmailDomain(t[r]) && r - at < 254) ++r;
|
||||
std::string_view dom = t.substr(at + 1, r - at - 1);
|
||||
auto dot = dom.rfind('.');
|
||||
if (dot == std::string_view::npos || dot == 0 || dot + 1 >= dom.size()) continue;
|
||||
bool tld_ok = dom.size() - dot - 1 >= 2;
|
||||
for (std::size_t i = dot + 1; tld_ok && i < dom.size(); ++i)
|
||||
if (!IsAlpha(dom[i])) tld_ok = false;
|
||||
if (!tld_ok) continue;
|
||||
out.push_back({l, r - l, pii::PiiClass::kEmail});
|
||||
if (out.size() >= kRedactMaxHitsPerCall) return;
|
||||
}
|
||||
}
|
||||
|
||||
inline void FindNationalId(std::string_view t, std::vector<Hit>& out) {
|
||||
for (std::size_t i = 0; i + 11 <= t.size(); ++i) {
|
||||
if (IsDigit(t[i]) && IsDigit(t[i + 1]) && IsDigit(t[i + 2]) && t[i + 3] == '-' &&
|
||||
IsDigit(t[i + 4]) && IsDigit(t[i + 5]) && t[i + 6] == '-' && IsDigit(t[i + 7]) &&
|
||||
IsDigit(t[i + 8]) && IsDigit(t[i + 9]) && IsDigit(t[i + 10]) &&
|
||||
(i == 0 || !IsDigit(t[i - 1])) && (i + 11 >= t.size() || !IsDigit(t[i + 11]))) {
|
||||
out.push_back({i, 11, pii::PiiClass::kNationalId});
|
||||
if (out.size() >= kRedactMaxHitsPerCall) return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inline void FindCard(std::string_view t, std::vector<Hit>& out) {
|
||||
for (std::size_t i = 0; i < t.size();) {
|
||||
if (!IsDigit(t[i])) {
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
std::size_t j = i;
|
||||
int digits = 0;
|
||||
while (j < t.size() && digits < 19 &&
|
||||
(IsDigit(t[j]) || t[j] == ' ' || t[j] == '-')) {
|
||||
if (IsDigit(t[j])) ++digits;
|
||||
++j;
|
||||
}
|
||||
while (j > i && !IsDigit(t[j - 1])) --j; // trailing separator trims
|
||||
if (digits >= 13 && digits <= 19 && j - i <= kMaxMatchLen &&
|
||||
(i == 0 || !IsDigit(t[i - 1]))) {
|
||||
out.push_back({i, j - i, pii::PiiClass::kFinancial});
|
||||
if (out.size() >= kRedactMaxHitsPerCall) return;
|
||||
}
|
||||
i = (j > i) ? j : i + 1;
|
||||
}
|
||||
}
|
||||
|
||||
inline void FindIban(std::string_view t, std::vector<Hit>& out) {
|
||||
for (std::size_t i = 0; i + 15 <= t.size(); ++i) {
|
||||
unsigned char a = t[i], b = t[i + 1];
|
||||
if (a < 'A' || a > 'Z' || b < 'A' || b > 'Z') continue;
|
||||
if (!IsDigit(t[i + 2]) || !IsDigit(t[i + 3])) continue;
|
||||
std::size_t j = i + 4;
|
||||
while (j < t.size() && j - i < 34 &&
|
||||
((t[j] >= 'A' && t[j] <= 'Z') || IsDigit(t[j])))
|
||||
++j;
|
||||
if (j - i >= 15) {
|
||||
out.push_back({i, j - i, pii::PiiClass::kFinancial});
|
||||
if (out.size() >= kRedactMaxHitsPerCall) return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inline void FindPhone(std::string_view t, std::vector<Hit>& out) {
|
||||
for (std::size_t i = 0; i < t.size();) {
|
||||
std::size_t s = i;
|
||||
if (t[s] == '+' && s + 1 < t.size() && IsDigit(t[s + 1])) ++s;
|
||||
if (!IsDigit(t[s])) {
|
||||
i = s + 1;
|
||||
continue;
|
||||
}
|
||||
std::size_t j = s;
|
||||
int digits = 0;
|
||||
while (j < t.size() && j - i < 40 && IsPhoneMid(t[j])) {
|
||||
if (IsDigit(t[j])) ++digits;
|
||||
++j;
|
||||
}
|
||||
while (j > s && !IsDigit(t[j - 1])) --j;
|
||||
if (digits >= 8 && j - i >= 8) {
|
||||
out.push_back({i, j - i, pii::PiiClass::kPhone});
|
||||
if (out.size() >= kRedactMaxHitsPerCall) return;
|
||||
}
|
||||
i = (j > i) ? j : i + 1;
|
||||
}
|
||||
}
|
||||
|
||||
inline void FindCredential(std::string_view t, std::vector<Hit>& out) {
|
||||
static constexpr const char* kKeys[] = {"password", "passwd", "secret",
|
||||
"api_key", "api-key", "apikey", "token"};
|
||||
for (std::size_t i = 0; i < t.size(); ++i) {
|
||||
for (const char* k : kKeys) {
|
||||
std::size_t kl = std::string_view(k).size();
|
||||
if (i + kl >= t.size()) continue;
|
||||
bool eq = true;
|
||||
for (std::size_t c = 0; eq && c < kl; ++c) eq = Lower(t[i + c]) == k[c];
|
||||
if (!eq) continue;
|
||||
if (i > 0 && (IsAlpha(t[i - 1]) || IsDigit(t[i - 1]) || t[i - 1] == '_')) continue;
|
||||
std::size_t j = i + kl;
|
||||
while (j < t.size() && t[j] == ' ') ++j;
|
||||
if (j >= t.size() || (t[j] != ':' && t[j] != '=')) continue;
|
||||
++j;
|
||||
while (j < t.size() && t[j] == ' ') ++j;
|
||||
std::size_t v = j;
|
||||
while (v < t.size() && t[v] != ' ' && t[v] != '\t' && t[v] != '\n' && t[v] != '\r' &&
|
||||
v - i < kMaxMatchLen)
|
||||
++v;
|
||||
if (v == j) continue;
|
||||
out.push_back({i, v - i, pii::PiiClass::kCredential});
|
||||
if (out.size() >= kRedactMaxHitsPerCall) return;
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace detail
|
||||
|
||||
struct Pattern {
|
||||
pii::PiiClass cls;
|
||||
std::function<void(std::string_view, std::vector<Hit>&)> find;
|
||||
};
|
||||
|
||||
// Priority order: specific shapes first so the phone matcher never eats an
|
||||
// SSN or card number.
|
||||
inline const std::vector<Pattern>& Patterns() {
|
||||
static const std::vector<Pattern> k = {
|
||||
{pii::PiiClass::kEmail, detail::FindEmail},
|
||||
{pii::PiiClass::kNationalId, detail::FindNationalId},
|
||||
{pii::PiiClass::kFinancial, detail::FindCard},
|
||||
{pii::PiiClass::kFinancial, detail::FindIban},
|
||||
{pii::PiiClass::kPhone, detail::FindPhone},
|
||||
{pii::PiiClass::kCredential, detail::FindCredential},
|
||||
};
|
||||
return k;
|
||||
}
|
||||
|
||||
// Collects hits across all patterns, earlier patterns winning overlaps.
|
||||
inline std::vector<Hit> FindHits(std::string_view text) {
|
||||
std::vector<Hit> acc;
|
||||
std::vector<char> covered(text.size(), 0);
|
||||
for (const auto& p : Patterns()) {
|
||||
std::vector<Hit> hs;
|
||||
p.find(text, hs);
|
||||
for (auto& h : hs) {
|
||||
if (acc.size() >= kRedactMaxHitsPerCall) return acc;
|
||||
bool overlap = false;
|
||||
for (std::size_t i = h.pos; i < h.pos + h.len && i < covered.size(); ++i)
|
||||
if (covered[i]) {
|
||||
overlap = true;
|
||||
break;
|
||||
}
|
||||
if (overlap) continue;
|
||||
for (std::size_t i = h.pos; i < h.pos + h.len && i < covered.size(); ++i) covered[i] = 1;
|
||||
acc.push_back(h);
|
||||
}
|
||||
}
|
||||
return acc;
|
||||
}
|
||||
|
||||
} // namespace molo::audit::redact
|
||||
@@ -0,0 +1,67 @@
|
||||
// Redactor: replaces PII in strings and log records with [redacted:<class>].
|
||||
// Wired into the logging Registry emit path and the error renderer by the
|
||||
// base patches (see PATCHES.md entries 3 and 4). Input is capped at
|
||||
// kRedactMaxBytesPerCall; oversize input is truncated before scanning.
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <variant>
|
||||
|
||||
#include "../adapt.h"
|
||||
#include "../caps.h"
|
||||
#include "pattern.h"
|
||||
|
||||
namespace molo::audit::redact {
|
||||
|
||||
class Redactor {
|
||||
public:
|
||||
// Scans text for every known PII shape and replaces each hit with
|
||||
// [redacted:<class>]. Pure function, safe to call from any thread.
|
||||
static std::string Redact(std::string_view text) {
|
||||
std::string in(text.substr(0, kRedactMaxBytesPerCall));
|
||||
std::vector<Hit> hits = FindHits(in);
|
||||
std::sort(hits.begin(), hits.end(),
|
||||
[](const Hit& a, const Hit& b) { return a.pos < b.pos; });
|
||||
std::string out;
|
||||
out.reserve(in.size() + hits.size() * 8);
|
||||
std::size_t last = 0;
|
||||
for (auto& h : hits) {
|
||||
out.append(in, last, h.pos - last);
|
||||
out += "[redacted:" + pii::ToString(h.cls) + "]";
|
||||
last = h.pos + h.len;
|
||||
}
|
||||
out.append(in, last, std::string::npos);
|
||||
return out;
|
||||
}
|
||||
|
||||
// Redacts one log record in place: message plus structured fields. Fields
|
||||
// whose key is sensitive are replaced wholesale; other string values are
|
||||
// scanned. Non-string values pass through (nothing to redact). Already
|
||||
// redacted values ("***" from the logger, "[redacted:*]" from an earlier
|
||||
// pass) are left alone so stacked redaction stays idempotent.
|
||||
// INTEGRATION: real FieldValue is a variant, not a string.
|
||||
static void RedactRecord(::molo::log::Record& r) {
|
||||
r.message = Redact(r.message);
|
||||
std::size_t n = 0;
|
||||
for (auto& f : r.fields) {
|
||||
if (++n > kRedactMaxFields) break;
|
||||
if (::molo::corr::is_sensitive_key(f.key)) {
|
||||
if (auto* s = std::get_if<std::string>(&f.value)) {
|
||||
if (*s != "***" && s->rfind("[redacted:", 0) != 0)
|
||||
*s = "[redacted:credential]";
|
||||
} else {
|
||||
f.value = std::string("[redacted:credential]");
|
||||
}
|
||||
} else if (auto* s = std::get_if<std::string>(&f.value)) {
|
||||
*s = Redact(*s);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Redacts an opaque error detail string for HTTP bodies and sinks.
|
||||
static std::string RedactDetail(std::string_view detail) { return Redact(detail); }
|
||||
};
|
||||
|
||||
} // namespace molo::audit::redact
|
||||
@@ -0,0 +1,93 @@
|
||||
// Enforcer: retention runs in two phases. DryRun scans rows through the
|
||||
// caller's scan callback and reports what would be deleted, held or kept,
|
||||
// without touching anything. Apply replays a DryRun report against the pinned
|
||||
// policy version and deletes through the caller's delete callback. Both are
|
||||
// synchronous functions for the supervisor scheduler, never request work.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "../caps.h"
|
||||
#include "hold.h"
|
||||
#include "policy.h"
|
||||
|
||||
namespace molo::audit::retention {
|
||||
|
||||
struct ScannedRow {
|
||||
std::string tenant;
|
||||
std::string table;
|
||||
std::string pk;
|
||||
std::uint32_t age_days = 0;
|
||||
};
|
||||
|
||||
struct DryRunReport {
|
||||
std::uint64_t policy_version = 0;
|
||||
std::uint64_t scanned = 0;
|
||||
std::uint64_t would_delete = 0;
|
||||
std::uint64_t held = 0;
|
||||
std::uint64_t kept = 0;
|
||||
bool truncated = false; // scan hit kRetentionScanMaxPerRun
|
||||
std::vector<ScannedRow> deletions; // exact rows apply() will delete
|
||||
};
|
||||
|
||||
struct ApplyResult {
|
||||
std::uint64_t policy_version = 0;
|
||||
std::uint64_t deleted = 0;
|
||||
std::uint64_t skipped_hold = 0;
|
||||
std::uint64_t skipped_policy = 0; // policy changed since dry run
|
||||
};
|
||||
|
||||
class Enforcer {
|
||||
public:
|
||||
using ScanFn = std::function<bool(ScannedRow&)>; // false ends the scan
|
||||
using DeleteFn = std::function<void(const ScannedRow&)>;
|
||||
|
||||
static DryRunReport DryRun(const Policy& policy, const Hold& hold, ScanFn scan) {
|
||||
DryRunReport r;
|
||||
r.policy_version = policy.version();
|
||||
ScannedRow row;
|
||||
while (scan(row)) {
|
||||
if (++r.scanned > kRetentionScanMaxPerRun) {
|
||||
r.truncated = true;
|
||||
break;
|
||||
}
|
||||
TableRule rule = policy.RuleFor(row.table);
|
||||
if (rule.ttl_days == 0 || row.age_days < rule.ttl_days + rule.grace_days) {
|
||||
++r.kept;
|
||||
continue;
|
||||
}
|
||||
if (hold.Has(Hold::RowKey(row.tenant, row.table, row.pk))) {
|
||||
++r.held;
|
||||
continue;
|
||||
}
|
||||
++r.would_delete;
|
||||
r.deletions.push_back(row);
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
static ApplyResult Apply(const Policy& policy, const Hold& hold, const DryRunReport& report,
|
||||
DeleteFn del) {
|
||||
ApplyResult r;
|
||||
r.policy_version = policy.version();
|
||||
for (const auto& row : report.deletions) {
|
||||
if (policy.version() != report.policy_version) {
|
||||
++r.skipped_policy;
|
||||
continue;
|
||||
}
|
||||
if (hold.Has(Hold::RowKey(row.tenant, row.table, row.pk))) {
|
||||
++r.skipped_hold;
|
||||
continue;
|
||||
}
|
||||
del(row);
|
||||
++r.deleted;
|
||||
}
|
||||
return r;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace molo::audit::retention
|
||||
@@ -0,0 +1,41 @@
|
||||
// Legal hold set: subjects or rows the Enforcer must never delete.
|
||||
// Capped at kMaxHolds; adding beyond the cap fails loudly so holds stay
|
||||
// explicit instead of silently dropping.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
|
||||
#include "../caps.h"
|
||||
|
||||
namespace molo::audit::retention {
|
||||
|
||||
class Hold {
|
||||
public:
|
||||
// Adds a hold on one key ("tenant:table:pk" or "tenant:subject:<id>").
|
||||
void Add(const std::string& key, const std::string& reason) {
|
||||
if (holds_.size() >= kMaxHolds && !holds_.count(key))
|
||||
throw std::runtime_error("hold set exceeds kMaxHolds");
|
||||
holds_[key] = reason;
|
||||
}
|
||||
|
||||
void Remove(const std::string& key) { holds_.erase(key); }
|
||||
bool Has(const std::string& key) const { return holds_.count(key) > 0; }
|
||||
std::size_t size() const { return holds_.size(); }
|
||||
|
||||
static std::string RowKey(const std::string& tenant, const std::string& table,
|
||||
const std::string& pk) {
|
||||
return tenant + ":" + table + ":" + pk;
|
||||
}
|
||||
|
||||
static std::string SubjectKey(const std::string& tenant, const std::string& subject) {
|
||||
return tenant + ":subject:" + subject;
|
||||
}
|
||||
|
||||
private:
|
||||
std::map<std::string, std::string> holds_;
|
||||
};
|
||||
|
||||
} // namespace molo::audit::retention
|
||||
@@ -0,0 +1,39 @@
|
||||
// Retention policy: per-table TTL in days plus a pinned version. Enforcement
|
||||
// always runs against one pinned version so a policy edit mid-run can never
|
||||
// mix two rule sets in a single report.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <string>
|
||||
|
||||
namespace molo::audit::retention {
|
||||
|
||||
struct TableRule {
|
||||
std::uint32_t ttl_days = 0; // 0 means keep forever
|
||||
std::uint32_t grace_days = 0; // extra days before apply() deletes
|
||||
};
|
||||
|
||||
class Policy {
|
||||
public:
|
||||
Policy(std::uint64_t version, std::map<std::string, TableRule> rules)
|
||||
: version_(version), rules_(std::move(rules)) {}
|
||||
|
||||
std::uint64_t version() const { return version_; }
|
||||
|
||||
TableRule RuleFor(const std::string& table) const {
|
||||
auto it = rules_.find(table);
|
||||
if (it != rules_.end()) return it->second;
|
||||
auto d = rules_.find("*");
|
||||
if (d != rules_.end()) return d->second;
|
||||
return TableRule{};
|
||||
}
|
||||
|
||||
const std::map<std::string, TableRule>& rules() const { return rules_; }
|
||||
|
||||
private:
|
||||
std::uint64_t version_;
|
||||
std::map<std::string, TableRule> rules_;
|
||||
};
|
||||
|
||||
} // namespace molo::audit::retention
|
||||
@@ -0,0 +1,82 @@
|
||||
// Eraser: erasure-request cascade across every store that can hold subject
|
||||
// data: sessions, cache tags, KV prefixes, ORM rows, uploads, tasks, and the
|
||||
// audit index itself. Each store binds through one deleter callback so this
|
||||
// header stays decoupled from the owning parts. Held subjects report Held and
|
||||
// delete nothing; statutorily exempt rows report Exempted per store; the audit
|
||||
// index row is never removed, it is rewritten to "erased:<request_id>".
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <string>
|
||||
|
||||
#include "../caps.h"
|
||||
#include "../retention/hold.h"
|
||||
#include "request.h"
|
||||
|
||||
namespace molo::audit::rights {
|
||||
|
||||
enum class EraseOutcome : std::uint8_t { kDone = 0, kHeld = 1, kExempted = 2 };
|
||||
|
||||
struct CascadeDeleters {
|
||||
// Each deleter removes subject rows from one store, returns erased count.
|
||||
std::function<std::uint64_t(const Subject&)> sessions;
|
||||
std::function<std::uint64_t(const Subject&)> cache_tags;
|
||||
std::function<std::uint64_t(const Subject&)> kv_prefix;
|
||||
std::function<std::uint64_t(const Subject&)> orm_rows;
|
||||
std::function<std::uint64_t(const Subject&)> uploads;
|
||||
std::function<std::uint64_t(const Subject&)> tasks;
|
||||
// Rewrites the audit index row for the subject to "erased:<request_id>".
|
||||
std::function<void(const Subject&, const std::string& request_id)> audit_index;
|
||||
// True when the subject carries rows the law forbids erasing.
|
||||
std::function<bool(const Subject&)> has_exemption;
|
||||
};
|
||||
|
||||
struct EraseReport {
|
||||
EraseOutcome outcome = EraseOutcome::kDone;
|
||||
std::uint64_t erased_total = 0;
|
||||
bool truncated = false; // hit kErasureMaxPerRun
|
||||
std::map<std::string, std::uint64_t> per_store;
|
||||
};
|
||||
|
||||
class Eraser {
|
||||
public:
|
||||
static std::string ErasedMarker(const std::string& request_id) {
|
||||
return "erased:" + request_id;
|
||||
}
|
||||
|
||||
static EraseReport Erase(const RightsRequest& req, const retention::Hold& hold,
|
||||
const CascadeDeleters& d) {
|
||||
EraseReport r;
|
||||
const std::string key =
|
||||
retention::Hold::SubjectKey(req.subject.tenant, req.subject.key);
|
||||
if (hold.Has(key)) {
|
||||
r.outcome = EraseOutcome::kHeld;
|
||||
return r;
|
||||
}
|
||||
auto run = [&](const std::string& name,
|
||||
const std::function<std::uint64_t(const Subject&)>& fn) {
|
||||
if (!fn || r.truncated) return;
|
||||
std::uint64_t n = fn(req.subject);
|
||||
if (r.erased_total + n > kErasureMaxPerRun) {
|
||||
r.truncated = true;
|
||||
return;
|
||||
}
|
||||
r.per_store[name] = n;
|
||||
r.erased_total += n;
|
||||
};
|
||||
run("sessions", d.sessions);
|
||||
run("cache_tags", d.cache_tags);
|
||||
run("kv_prefix", d.kv_prefix);
|
||||
run("orm_rows", d.orm_rows);
|
||||
run("uploads", d.uploads);
|
||||
run("tasks", d.tasks);
|
||||
if (d.audit_index && !r.truncated) d.audit_index(req.subject, req.id);
|
||||
if (d.has_exemption && d.has_exemption(req.subject))
|
||||
r.outcome = EraseOutcome::kExempted;
|
||||
return r;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace molo::audit::rights
|
||||
@@ -0,0 +1,96 @@
|
||||
// Exporter: builds a deterministic subject bundle for access/export
|
||||
// requests. Rows arrive through the caller's row callback in any order; the
|
||||
// bundle sorts them, splits them into capped chunks, and ships a manifest
|
||||
// with row counts plus a SHA256 checksum per chunk and over the whole.
|
||||
// Capped at kExportMaxRows and kExportMaxBytes; exceeding either fails the
|
||||
// export loudly instead of shipping a partial bundle.
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "../adapt.h"
|
||||
#include "../caps.h"
|
||||
#include "request.h"
|
||||
|
||||
namespace molo::audit::rights {
|
||||
|
||||
struct ExportRow {
|
||||
std::string table;
|
||||
std::string pk;
|
||||
std::string json; // already redacted/filtered to exportable marks
|
||||
};
|
||||
|
||||
struct ExportChunk {
|
||||
std::uint32_t index = 0;
|
||||
std::string bytes; // newline-delimited row JSON
|
||||
std::string sha256;
|
||||
};
|
||||
|
||||
struct ExportBundle {
|
||||
std::string request_id;
|
||||
std::string manifest_json;
|
||||
std::vector<ExportChunk> chunks;
|
||||
std::string bundle_sha256;
|
||||
std::uint64_t rows = 0;
|
||||
};
|
||||
|
||||
class Exporter {
|
||||
public:
|
||||
using RowFn = std::function<bool(ExportRow&)>; // false ends input
|
||||
static constexpr std::size_t kChunkBytes = 1024 * 1024;
|
||||
|
||||
static ExportBundle Build(const RightsRequest& req, RowFn rows) {
|
||||
std::vector<ExportRow> all;
|
||||
ExportRow r;
|
||||
while (rows(r)) {
|
||||
all.push_back(r);
|
||||
if (all.size() > kExportMaxRows)
|
||||
throw std::runtime_error("export exceeds kExportMaxRows");
|
||||
}
|
||||
std::sort(all.begin(), all.end(), [](const ExportRow& a, const ExportRow& b) {
|
||||
if (a.table != b.table) return a.table < b.table;
|
||||
return a.pk < b.pk;
|
||||
});
|
||||
ExportBundle b;
|
||||
b.request_id = req.id;
|
||||
b.rows = all.size();
|
||||
std::string cur;
|
||||
std::uint32_t idx = 0;
|
||||
std::size_t total = 0;
|
||||
auto flush = [&] {
|
||||
if (cur.empty()) return;
|
||||
ExportChunk c;
|
||||
c.index = idx++;
|
||||
c.bytes = cur;
|
||||
c.sha256 = ::molo::security::crypto::Sha256::hex(
|
||||
::molo::security::crypto::Sha256::hash(cur));
|
||||
total += cur.size();
|
||||
if (total > kExportMaxBytes) throw std::runtime_error("export exceeds kExportMaxBytes");
|
||||
b.chunks.push_back(std::move(c));
|
||||
cur.clear();
|
||||
};
|
||||
for (auto& row : all) {
|
||||
std::string line = "{\"table\":\"" + row.table + "\",\"pk\":\"" + row.pk +
|
||||
"\",\"row\":" + row.json + "}\n";
|
||||
if (cur.size() + line.size() > kChunkBytes) flush();
|
||||
cur += line;
|
||||
}
|
||||
flush();
|
||||
std::string all_hashes;
|
||||
for (auto& c : b.chunks) all_hashes += c.sha256;
|
||||
b.bundle_sha256 = ::molo::security::crypto::Sha256::hex(
|
||||
::molo::security::crypto::Sha256::hash(all_hashes));
|
||||
b.manifest_json = "{\"request\":\"" + req.id + "\",\"rows\":" + std::to_string(b.rows) +
|
||||
",\"chunks\":" + std::to_string(b.chunks.size()) + ",\"sha256\":\"" +
|
||||
b.bundle_sha256 + "\"}";
|
||||
return b;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace molo::audit::rights
|
||||
@@ -0,0 +1,107 @@
|
||||
// RightsRequest: one data-subject request lifecycle. States run received,
|
||||
// verified, approved, in_progress, fulfilled, rejected, expired. Due date is
|
||||
// created + 30 days; one extension moves it to + 60 days. Open-request counts
|
||||
// per tenant are capped at kMaxOpenRequestsPerTenant by the caller-owned
|
||||
// RequestBook below.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "../caps.h"
|
||||
#include "subject.h"
|
||||
|
||||
namespace molo::audit::rights {
|
||||
|
||||
enum class RequestKind : std::uint8_t { kAccess = 0, kExport = 1, kErasure = 2 };
|
||||
|
||||
enum class RequestState : std::uint8_t {
|
||||
kReceived = 0,
|
||||
kVerified = 1,
|
||||
kApproved = 2,
|
||||
kInProgress = 3,
|
||||
kFulfilled = 4,
|
||||
kRejected = 5,
|
||||
kExpired = 6,
|
||||
};
|
||||
|
||||
inline constexpr std::int64_t kDueDays = 30;
|
||||
inline constexpr std::int64_t kExtendedDueDays = 60;
|
||||
inline constexpr std::int64_t kDaySeconds = 86400;
|
||||
|
||||
struct RightsRequest {
|
||||
std::string id;
|
||||
Subject subject;
|
||||
RequestKind kind = RequestKind::kAccess;
|
||||
RequestState state = RequestState::kReceived;
|
||||
std::int64_t created_ts = 0;
|
||||
std::int64_t due_ts = 0; // created + 30d, or + 60d once extended
|
||||
bool extended = false;
|
||||
|
||||
bool Open() const {
|
||||
return state == RequestState::kReceived || state == RequestState::kVerified ||
|
||||
state == RequestState::kApproved || state == RequestState::kInProgress;
|
||||
}
|
||||
};
|
||||
|
||||
class RequestBook {
|
||||
public:
|
||||
RightsRequest& Open(std::string id, Subject subject, RequestKind kind, std::int64_t now) {
|
||||
if (OpenCount(subject.tenant) >= kMaxOpenRequestsPerTenant)
|
||||
throw std::runtime_error("tenant exceeds kMaxOpenRequestsPerTenant");
|
||||
RightsRequest r;
|
||||
r.id = id;
|
||||
r.subject = std::move(subject);
|
||||
r.kind = kind;
|
||||
r.created_ts = now;
|
||||
r.due_ts = now + kDueDays * kDaySeconds;
|
||||
reqs_[r.id] = r;
|
||||
return reqs_[r.id];
|
||||
}
|
||||
|
||||
void Transition(const std::string& id, RequestState to) {
|
||||
auto it = reqs_.find(id);
|
||||
if (it == reqs_.end()) throw std::runtime_error("unknown rights request: " + id);
|
||||
it->second.state = to;
|
||||
}
|
||||
|
||||
void ExtendOnce(const std::string& id) {
|
||||
auto it = reqs_.find(id);
|
||||
if (it == reqs_.end()) throw std::runtime_error("unknown rights request: " + id);
|
||||
if (it->second.extended) throw std::runtime_error("request already extended: " + id);
|
||||
it->second.extended = true;
|
||||
it->second.due_ts = it->second.created_ts + kExtendedDueDays * kDaySeconds;
|
||||
}
|
||||
|
||||
// Marks overdue open requests expired. Returns the expired ids.
|
||||
std::vector<std::string> ExpireOverdue(std::int64_t now) {
|
||||
std::vector<std::string> out;
|
||||
for (auto& [id, r] : reqs_)
|
||||
if (r.Open() && now > r.due_ts) {
|
||||
r.state = RequestState::kExpired;
|
||||
out.push_back(id);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::size_t OpenCount(const std::string& tenant) const {
|
||||
std::size_t n = 0;
|
||||
for (auto& [_, r] : reqs_)
|
||||
if (r.subject.tenant == tenant && r.Open()) ++n;
|
||||
return n;
|
||||
}
|
||||
|
||||
const RightsRequest& Get(const std::string& id) const {
|
||||
auto it = reqs_.find(id);
|
||||
if (it == reqs_.end()) throw std::runtime_error("unknown rights request: " + id);
|
||||
return it->second;
|
||||
}
|
||||
|
||||
private:
|
||||
std::map<std::string, RightsRequest> reqs_;
|
||||
};
|
||||
|
||||
} // namespace molo::audit::rights
|
||||
@@ -0,0 +1,20 @@
|
||||
// Data subject identity for rights requests: tenant plus one stable subject
|
||||
// key (user id, account id). Key format is "<kind>:<id>", e.g. "user:42".
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace molo::audit::rights {
|
||||
|
||||
struct Subject {
|
||||
std::string tenant;
|
||||
std::string key;
|
||||
|
||||
static Subject User(const std::string& tenant, const std::string& id) {
|
||||
return Subject{tenant, "user:" + id};
|
||||
}
|
||||
|
||||
std::string ToString() const { return tenant + ":" + key; }
|
||||
};
|
||||
|
||||
} // namespace molo::audit::rights
|
||||
Reference in New Issue
Block a user